promptgraph-mcp 2.4.8 → 2.6.0

package/indexer.js

@@ -3,7 +3,8 @@ import { createHash } from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import { parseSkillFile, isSkillFile, filterWithClassifier } from './parser.js';
-import { embedBatch, cosineSimilarity, BATCH_SIZE } from './embedder.js';
+import { embedBatch, cosineSimilarity } from './embedder.js';
+import { BATCH_SIZE } from './config.js';
 import { getDb, skillId, vecToBlob } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
@@ -18,7 +19,7 @@ function sanitizePath(filePath) {
   return path.resolve(filePath);
 }
 
-async function indexBatch(db, skills, { fast = false } = {}) {
+export async function indexBatch(db, skills, { fast = false } = {}) {
   const upsertSkill = db.prepare(`
     INSERT INTO skills (id, name, description, path, source, content, hash, version, author, license, updated_at, downloads, verified)
     VALUES (@id, @name, @description, @path, @source, @content, @hash, @version, @author, @license, @updated_at, @downloads, @verified)
@@ -106,21 +107,15 @@ export async function indexAll({ fast = false } = {}) {
     normDir: path.resolve(s.dir),
   })).sort((a, b) => b.normDir.length - a.normDir.length); // longest first
 
-  console.error('DEBUG normalizedSources count:', normalizedSources.length);
-  console.error('DEBUG first dir:', normalizedSources[0]?.dir);
-
   const seenFiles = new Set();
   const allFiles = [];
   for (const { dir, source } of normalizedSources) {
     const files = globSync(`${dir}/**/*.md`);
-    console.error('DEBUG source:', source, 'dir:', dir, 'count:', files.length);
     for (const f of files) {
       const norm = sanitizePath(f);
       if (!seenFiles.has(norm)) {
         seenFiles.add(norm);
         allFiles.push({ file: norm, source });
-      } else {
-        console.error('DEBUG duplicate:', norm);
       }
     }
     if (allFiles.length > MAX_FILE_COUNT) {
@@ -129,7 +124,6 @@ export async function indexAll({ fast = false } = {}) {
     }
   }
   const total = allFiles.length;
-  console.error('DEBUG total files found:', total);
   info(`Found ${chalk.white.bold(total)} files`);
 
   // reconcile: remove skills whose files no longer exist OR whose name changed
@@ -210,6 +204,7 @@ export async function indexAll({ fast = false } = {}) {
         batch = [];
         const eta = count > 0 ? Math.round((total - count) * (Date.now() - start) / count / 1000) : '?';
         progress(count, total, { skipped, eta, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) {
       errors++;
@@ -301,6 +296,7 @@ export async function indexSource(dir, sourceName) {
         await indexBatch(db, batch);
         count += batch.length; batch = [];
         progress(count, total, { skipped, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) { errors++; count++; }
   }

package/marketplace.js

@@ -61,6 +61,16 @@ function validateRegistryEntry(item) {
   }
   if (item.downloads !== undefined && typeof item.downloads !== 'number') errors.push('downloads must be a number');
   if (item.verified !== undefined && typeof item.verified !== 'boolean') errors.push('verified must be a boolean');
+  if (item.trustLevel !== undefined && typeof item.trustLevel !== 'string') errors.push('trustLevel must be a string');
+  if (item.rating !== undefined) {
+    if (typeof item.rating !== 'number') errors.push('rating must be a number');
+    else if (item.rating < 0 || item.rating > 5) errors.push('rating must be between 0 and 5');
+  }
+  if (item.popularity !== undefined && typeof item.popularity !== 'number') errors.push('popularity must be a number');
+  if (item.lastUpdate !== undefined) {
+    if (typeof item.lastUpdate !== 'string') errors.push('lastUpdate must be a string');
+    else if (isNaN(Date.parse(item.lastUpdate))) errors.push(`lastUpdate "${item.lastUpdate}" is not a valid ISO date`);
+  }
   if (errors.length > 0) {
     console.warn(`[PromptGraph] Skipping invalid registry entry "${item.id || item.name || '(unnamed)'}": ${errors.join('; ')}`);
     return { ok: false };
@@ -162,7 +172,7 @@ export async function browseMarketplace(topK = 20) {
     const registry = JSON.parse(text);
     if (!Array.isArray(registry.skills)) return { error: 'Invalid registry format' };
     return registry.skills
-      .map(s => ({ ...s, code: s.code || codeFor(s.id) })) // auto-fill code
+      .map(s => Object.assign(Object.create(null), s, { code: s.code || codeFor(s.id) }))
       .filter(s => validateRegistryEntry(s).ok)
       .filter(s => s.raw_url)
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
@@ -287,7 +297,7 @@ export async function browseBundles(topK = 20) {
 
     return bundles
       .filter(b => !b.repo_url || b.skillCount > 0)
-      .map(b => ({ ...b, code: b.code || codeFor(b.id) }))
+      .map(b => Object.assign(Object.create(null), b, { code: b.code || codeFor(b.id) }))
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
   } catch (e) {
@@ -548,3 +558,74 @@ export function pruneInvalidRepos() {
   saveConfig(config);
   return { removed, kept, errors };
 }
+
+// ── Trust level system ─────────────────────────────────────────────────────────
+const VALID_TRUST_LEVELS = ['verified', 'official', 'community', 'trusted', 'unknown']
+
+function calcPopularity(downloads = 0, rating = 0) {
+  return Math.round((downloads || 0) * ((rating || 0) + 1) * 100) / 100
+}
+
+export async function setTrustLevel(name, level) {
+  const db = getDb()
+  if (!VALID_TRUST_LEVELS.includes(level)) {
+    return { ok: false, error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  const id = String(name)
+  db.prepare(`
+    INSERT INTO registry_entries (id, trust_level, last_update)
+    VALUES (?, ?, datetime('now'))
+    ON CONFLICT(id) DO UPDATE SET trust_level = excluded.trust_level, last_update = excluded.last_update
+  `).run(id, level)
+  return { ok: true }
+}
+
+export async function getByTrustLevel(level) {
+  const db = getDb()
+  if (level && !VALID_TRUST_LEVELS.includes(level)) {
+    return { error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  if (level) {
+    return db.prepare('SELECT * FROM registry_entries WHERE trust_level = ? ORDER BY id').all(level)
+  }
+  return db.prepare('SELECT * FROM registry_entries ORDER BY id').all()
+}
+
+export async function incrementDownloads(name) {
+  const db = getDb()
+  const id = String(name)
+  const existing = db.prepare('SELECT downloads, rating FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const newDownloads = (existing.downloads || 0) + 1
+    const pop = calcPopularity(newDownloads, existing.rating || 0)
+    db.prepare('UPDATE registry_entries SET downloads = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newDownloads, pop, id)
+  } else {
+    const pop = calcPopularity(1, 0)
+    db.prepare('INSERT INTO registry_entries (id, downloads, popularity, last_update) VALUES (?, 1, ?, datetime(\'now\'))')
+      .run(id, pop)
+  }
+  return { ok: true }
+}
+
+export async function rateSkill(name, rating) {
+  const db = getDb()
+  if (typeof rating !== 'number' || rating < 0 || rating > 5) {
+    return { ok: false, error: 'Rating must be a number between 0 and 5' }
+  }
+  const id = String(name)
+  const existing = db.prepare('SELECT rating, rating_count, downloads FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const count = existing.rating_count || 0
+    const newRating = Math.round(((existing.rating || 0) * count + rating) / (count + 1) * 100) / 100
+    const newCount = count + 1
+    const pop = calcPopularity(existing.downloads || 0, newRating)
+    db.prepare('UPDATE registry_entries SET rating = ?, rating_count = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newRating, newCount, pop, id)
+  } else {
+    const pop = calcPopularity(0, rating)
+    db.prepare('INSERT INTO registry_entries (id, rating, rating_count, popularity, last_update) VALUES (?, ?, 1, ?, datetime(\'now\'))')
+      .run(id, rating, pop)
+  }
+  return { ok: true }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.8",
+  "version": "2.6.0",
   "files": [
     "*.js",
     "src/**/*.js",
@@ -55,6 +55,6 @@
     "vitest": "^4.1.8"
   },
   "overrides": {
-    "tar": "^6.2.1"
+    "tar": "^7.5.11"
   }
 }

package/search.js

@@ -2,24 +2,10 @@ import { embed, cosineSimilarity } from './embedder.js';
 import { getDb, blobToVec } from './db.js';
 import { annSearch } from './ann.js';
 
-let embedWeight = 0.7;
-let bm25Weight = 0.3;
-
-export function setHybridWeights(ew, bw) {
-  embedWeight = ew;
-  bm25Weight = bw;
-}
-
-function adaptWeights(query) {
-  // Technical terms (caps, digits) → BM25 matters more
+function getHybridWeights(query) {
   const hasTechTerms = /[A-Z]|\d/.test(query);
-  if (hasTechTerms) {
-    embedWeight = 0.5;
-    bm25Weight = 0.5;
-  } else {
-    embedWeight = 0.7;
-    bm25Weight = 0.3;
-  }
+  if (hasTechTerms) return { embedWeight: 0.5, bm25Weight: 0.5 }
+  return { embedWeight: 0.7, bm25Weight: 0.3 }
 }
 
 function applyRatingBoost(db, id, score) {
@@ -82,7 +68,7 @@ function runBM25Search(db, query, topK) {
 
 export async function search(query, topK = 5) {
   const db = getDb();
-  adaptWeights(query);
+  const { embedWeight, bm25Weight } = getHybridWeights(query);
   const queryVec = await embed(query);
 
   const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
@@ -116,13 +102,29 @@ export async function search(query, topK = 5) {
     }
   }
 
-  return [...combined.entries()]
+  const ordered = [...combined.entries()]
     .map(([id, s]) => ({
       id,
       score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
     }))
     .sort((a, b) => b.score - a.score)
-    .slice(0, topK)
+
+  const rerankerEnabled = process.env.PG_RERANKER !== '0'
+
+  if (rerankerEnabled) {
+    const { Reranker } = await import('./src/reranker/reranker.js')
+    const reranker = new Reranker()
+    const topN = ordered.slice(0, 20)
+      .map(({ id, score }) => {
+        const s = skillWithSnippet(db, id, score)
+        return s ? { id, text: s.snippet, score } : null
+      })
+      .filter(Boolean)
+    const reranked = await reranker.rerank(query, topN)
+    return reranked.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score))).filter(Boolean)
+  }
+
+  return ordered.slice(0, topK)
     .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
     .filter(Boolean);
 }

package/src/reranker/reranker.js

@@ -0,0 +1,28 @@
+export class Reranker {
+  constructor(modelName = 'default', device = 'cpu') {
+    this.modelName = modelName
+    this.device = device
+  }
+
+  // Term-overlap boost on top of hybrid search scores
+  // This is a lightweight reranker that ranks by term overlap ratio.
+  // Replace with BGE Reranker / MiniLM cross-encoder via ONNX for deeper semantic reranking.
+  async rerank(query, results) {
+    if (!results || results.length === 0) return []
+
+    const queryTerms = (query.toLowerCase().match(/\w+/g) || []).slice(0, 50)
+    const queryTermCount = queryTerms.length || 1
+
+    const scored = results.map(r => {
+      const text = (r.text || '').toLowerCase()
+      const termOverlap = queryTerms.filter(t => text.includes(t)).length / queryTermCount
+
+      return {
+        ...r,
+        score: 0.8 * (r.score || 0) + 0.2 * termOverlap,
+      }
+    })
+
+    return scored.sort((a, b) => b.score - a.score).slice(0, 5)
+  }
+}

package/src/utils/rate-limiter.js

@@ -0,0 +1,33 @@
+export class RateLimiter {
+  constructor({ maxRequests, windowMs }) {
+    this.maxRequests = maxRequests
+    this.windowMs = windowMs
+    this.timestamps = []
+    this._chain = Promise.resolve()
+  }
+
+  async acquire() {
+    const prev = this._chain
+    this._chain = prev.then(() => this._doAcquire())
+    return this._chain
+  }
+
+  async _doAcquire() {
+    while (!this.tryAcquire()) {
+      const oldest = this.timestamps[0]
+      const waitMs = oldest
+        ? Math.max(1, oldest + this.windowMs - Date.now())
+        : 100
+      await new Promise(r => setTimeout(r, Math.min(waitMs, this.windowMs)))
+    }
+  }
+
+  tryAcquire() {
+    const now = Date.now()
+    const cutoff = now - this.windowMs
+    this.timestamps = this.timestamps.filter(t => t > cutoff)
+    if (this.timestamps.length >= this.maxRequests) return false
+    this.timestamps.push(now)
+    return true
+  }
+}

package/validator.js

@@ -1,5 +1,7 @@
 import fs from 'fs';
+import path from 'path';
 import matter from 'gray-matter';
+import { MAX_DOWNLOAD_SIZE } from './config.js';
 
 // patterns that indicate malicious or junk skills
 const DANGEROUS_PATTERNS = [
@@ -86,6 +88,19 @@ export function validateSkill(filePath) {
   if (pathParts.includes('..')) {
     errors.push('Path traversal detected: file path contains ".."');
   }
+
+  // extension whitelist — reject unexpected file types
+  const ALLOWED_EXTENSIONS = new Set(['.md', '.json', '.yaml', '.yml', '.txt', '.js', '.ts', '.py', '.rs', '.toml']);
+  const ext = path.extname(filePath).toLowerCase();
+  if (!ALLOWED_EXTENSIONS.has(ext)) {
+    errors.push(`File extension "${ext}" not allowed. Allowed: ${[...ALLOWED_EXTENSIONS].join(', ')}`);
+  }
+
+  // binary content detection — must be valid UTF-8 with no null bytes
+  if (raw.includes('\0')) {
+    errors.push('File contains null bytes (binary content)');
+  }
+
   if (['readme.md', 'changelog.md', 'license.md', 'contributing.md'].includes(base)) {
     warnings.push('Filename looks like a docs file, not a skill.');
   }
@@ -142,6 +157,15 @@ export function validateBundle(def) {
   return { ok: errors.length === 0, errors, warnings };
 }
 
+export function sanitizeExternalContent(content) {
+  if (typeof content !== 'string') return ''
+  let sanitized = content.replace(/\0/g, '')
+  if (Buffer.byteLength(sanitized, 'utf8') > MAX_DOWNLOAD_SIZE) {
+    sanitized = Buffer.from(sanitized, 'utf8').subarray(0, MAX_DOWNLOAD_SIZE).toString('utf8')
+  }
+  return sanitized
+}
+
 // CLI: node validator.js <file>
 if (import.meta.url === `file://${process.argv[1]}`) {
   const file = process.argv[2];

package/src/filter/cluster.js

@@ -1,52 +0,0 @@
-import { cosineSimilarity } from '../../embedder.js';
-
-const INFERENCE_THRESHOLD = 0.35;
-
-export function findClusters(skills, embeddings) {
-  if (!skills.length || !embeddings) return [];
-  const clusters = [];
-  const assigned = new Set();
-
-  for (let i = 0; i < skills.length; i++) {
-    if (assigned.has(i)) continue;
-    const cluster = { centroid: embeddings[i], members: [skills[i]], indices: [i] };
-    assigned.add(i);
-
-    for (let j = i + 1; j < skills.length; j++) {
-      if (assigned.has(j)) continue;
-      if (cosineSimilarity(embeddings[i], embeddings[j]) >= INFERENCE_THRESHOLD) {
-        cluster.members.push(skills[j]);
-        cluster.indices.push(j);
-        assigned.add(j);
-      }
-    }
-
-    clusters.push(cluster);
-  }
-
-  return clusters;
-}
-
-export function expandResults(annResults, db) {
-  if (!annResults || annResults.length === 0) return annResults;
-
-  const seenIds = new Set();
-  const expanded = [];
-
-  for (const r of annResults) {
-    if (!seenIds.has(r.skill_id)) {
-      seenIds.add(r.skill_id);
-      expanded.push(r);
-    }
-
-    const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(r.skill_id);
-    for (const c of callees) {
-      if (!seenIds.has(c.to_skill)) {
-        seenIds.add(c.to_skill);
-        expanded.push({ skill_id: c.to_skill, score: r.score * 0.85 });
-      }
-    }
-  }
-
-  return expanded.sort((a, b) => b.score - a.score);
-}

package/src/filter/dedup.js

@@ -1,36 +0,0 @@
-import { cosineSimilarity } from '../../embedder.js';
-
-const DEDUP_THRESHOLD = 0.97;
-
-export function dedup(skills, embeddings) {
-  if (!skills.length) return { skills: [], embeddings: [] };
-  if (embeddings && embeddings.length !== skills.length) {
-    throw new Error('embeddings length must match skills length');
-  }
-
-  const keep = [true]; // first skill always kept
-  const kept = [0];
-
-  for (let i = 1; i < skills.length; i++) {
-    let dup = false;
-    if (embeddings) {
-      for (const j of kept) {
-        if (cosineSimilarity(embeddings[i], embeddings[j]) >= DEDUP_THRESHOLD) {
-          dup = true;
-          break;
-        }
-      }
-    }
-    if (dup) {
-      keep.push(false);
-    } else {
-      keep.push(true);
-      kept.push(i);
-    }
-  }
-
-  return {
-    skills: skills.filter((_, i) => keep[i]),
-    embeddings: embeddings ? embeddings.filter((_, i) => keep[i]) : undefined,
-  };
-}