promptgraph-mcp 2.4.8 → 2.5.0

package/config.js

@@ -8,6 +8,13 @@ export const PROMPTGRAPH_DIR = path.join(CLAUDE_DIR, '.promptgraph');
 export const SKILLS_STORE_DIR = path.join(CLAUDE_DIR, 'skills-store');
 const CONFIG_PATH = path.join(PROMPTGRAPH_DIR, 'config.json');
 
+export const MAX_DOWNLOAD_SIZE = 50 * 1024 * 1024  // 50 MB per file
+export const MAX_FILE_COUNT = 50000                  // max files per repo
+export const MAX_REPO_SIZE = 500 * 1024 * 1024      // 500 MB per repo
+export const RATE_LIMIT_REQUESTS = 30               // requests per window
+export const RATE_LIMIT_WINDOW_MS = 60000           // 1 minute window
+export const BATCH_SIZE = 100                       // batch indexing size
+
 const DEFAULTS = {
   sources: [
     { dir: path.join(CLAUDE_DIR, 'skills-store'), source: 'skills-store' },

package/db.js

@@ -82,6 +82,34 @@ export function getDb() {
   if (!cols.includes('verified')) {
     db.exec('ALTER TABLE skills ADD COLUMN verified INTEGER DEFAULT 0');
   }
+  if (!cols.includes('trust_level')) {
+    db.exec('ALTER TABLE skills ADD COLUMN trust_level TEXT DEFAULT \'unknown\'');
+  }
+  if (!cols.includes('rating')) {
+    db.exec('ALTER TABLE skills ADD COLUMN rating REAL DEFAULT 0');
+  }
+  if (!cols.includes('rating_count')) {
+    db.exec('ALTER TABLE skills ADD COLUMN rating_count INTEGER DEFAULT 0');
+  }
+  if (!cols.includes('popularity')) {
+    db.exec('ALTER TABLE skills ADD COLUMN popularity REAL DEFAULT 0');
+  }
+  if (!cols.includes('last_update')) {
+    db.exec('ALTER TABLE skills ADD COLUMN last_update TEXT');
+  }
+
+  // migrate: registry entries metadata table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS registry_entries (
+      id TEXT PRIMARY KEY,
+      trust_level TEXT DEFAULT 'unknown',
+      downloads INTEGER DEFAULT 0,
+      rating REAL DEFAULT 0,
+      rating_count INTEGER DEFAULT 0,
+      popularity REAL DEFAULT 0,
+      last_update TEXT
+    );
+  `);
 
   // migrate: convert JSON text embeddings to Float32 BLOB (one-time, ~10x smaller)
   const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();

package/github-import.js

@@ -4,15 +4,60 @@ import fs from 'fs';
 import https from 'https';
 import { globSync } from 'glob';
 import { indexAll, indexSource } from './indexer.js';
-import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './config.js';
+import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR, MAX_DOWNLOAD_SIZE, MAX_FILE_COUNT, MAX_REPO_SIZE, RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_MS } from './config.js';
 import { validateSkill } from './validator.js';
 import { isSkillFile } from './parser.js';
+import { RateLimiter } from './src/utils/rate-limiter.js';
+
+const githubRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS, windowMs: RATE_LIMIT_WINDOW_MS })
+const downloadRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS * 2, windowMs: RATE_LIMIT_WINDOW_MS })
 
 const SKILL_DIRS = ['skills', 'commands', 'prompts', 'agents', 'skills-store', 'slash-commands', 'custom-commands', 'templates'];
 
 // ── helpers ───────────────────────────────────────────────────────────────────
 
-function httpsGet(url) {
+const repoStats = new Map()
+
+function getRepoStats(ownerRepo) {
+  if (!repoStats.has(ownerRepo)) {
+    repoStats.set(ownerRepo, { totalBytes: 0, fileCount: 0 })
+  }
+  return repoStats.get(ownerRepo)
+}
+
+function streamDownload(url, maxSize = MAX_DOWNLOAD_SIZE) {
+  return new Promise((res, rej) => {
+    const token = process.env.GITHUB_TOKEN;
+    const headers = { 'User-Agent': 'promptgraph-mcp' };
+    if (token && url.startsWith('https://raw.')) headers['Authorization'] = `Bearer ${token}`;
+    const req = https.get(url, { headers }, r => {
+      if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location)
+        return streamDownload(r.headers.location, maxSize).then(res, rej);
+      if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
+      const cl = parseInt(r.headers['content-length'], 10);
+      if (!isNaN(cl) && cl > maxSize) {
+        r.resume();
+        return rej(new Error(`Content-Length ${cl} exceeds max ${maxSize}`));
+      }
+      let d = ''
+      let total = 0
+      r.setEncoding('utf8')
+      r.on('data', c => {
+        total += Buffer.byteLength(c, 'utf8')
+        if (total > maxSize) {
+          r.destroy()
+          return rej(new Error(`Download exceeded ${maxSize} bytes`))
+        }
+        d += c
+      })
+      r.on('end', () => res(d))
+    })
+    req.on('error', rej)
+  })
+}
+
+async function httpsGet(url) {
+  await githubRateLimiter.acquire()
   const token = process.env.GITHUB_TOKEN;
   const headers = { 'User-Agent': 'promptgraph-mcp' };
   if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
@@ -39,11 +84,22 @@ function repoExists(ownerRepo) {
 }
 
 // Download one .md file, run validateSkill on it, return errors/warnings.
-async function validateMdFile(file, tmpDir) {
+async function validateMdFile(file, tmpDir, ownerRepo) {
   const errors = [];
   const warnings = [];
+  const stats = getRepoStats(ownerRepo);
   try {
-    const content = await httpsGet(file.download_url);
+    if (stats.fileCount >= MAX_FILE_COUNT) {
+      errors.push(`${file.name}: skipped — repo file count limit (${MAX_FILE_COUNT}) reached`);
+      return { errors, warnings };
+    }
+    await downloadRateLimiter.acquire()
+    const content = await streamDownload(file.download_url);
+    stats.totalBytes += Buffer.byteLength(content, 'utf8');
+    stats.fileCount++;
+    if (stats.totalBytes > MAX_REPO_SIZE) {
+      return { errors: [...errors, `${file.name}: repo size limit (${MAX_REPO_SIZE}) exceeded`], warnings };
+    }
     const tmpPath = path.join(tmpDir, file.name);
     fs.mkdirSync(path.dirname(tmpPath), { recursive: true });
     fs.writeFileSync(tmpPath, content);
@@ -119,7 +175,7 @@ export async function validateRepoSkills(ownerRepo) {
   let warnings = [];
 
   for (const file of mdToValidate) {
-    const r = await validateMdFile(file, tmpDir);
+    const r = await validateMdFile(file, tmpDir, ownerRepo);
     errors.push(...r.errors);
     warnings.push(...r.warnings);
   }
@@ -464,6 +520,10 @@ export async function importFromGitHub(repoUrl) {
   const label = skillsSubdir || localLabel;
   const mdFiles = globSync(`${skillsDir}/**/*.md`);
 
+  if (mdFiles.length > MAX_FILE_COUNT) {
+    console.warn(`Warning: ${mdFiles.length} .md files exceeds limit of ${MAX_FILE_COUNT} — truncating`);
+  }
+
   if (skillsSubdir) {
     console.log(`Sparse-checkout: ${label}/ only (${mdFiles.length} .md files, no other repo files)`);
   } else {

package/indexer.js

@@ -3,7 +3,8 @@ import { createHash } from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import { parseSkillFile, isSkillFile, filterWithClassifier } from './parser.js';
-import { embedBatch, cosineSimilarity, BATCH_SIZE } from './embedder.js';
+import { embedBatch, cosineSimilarity } from './embedder.js';
+import { BATCH_SIZE } from './config.js';
 import { getDb, skillId, vecToBlob } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
@@ -210,6 +211,7 @@ export async function indexAll({ fast = false } = {}) {
         batch = [];
         const eta = count > 0 ? Math.round((total - count) * (Date.now() - start) / count / 1000) : '?';
         progress(count, total, { skipped, eta, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) {
       errors++;
@@ -301,6 +303,7 @@ export async function indexSource(dir, sourceName) {
         await indexBatch(db, batch);
         count += batch.length; batch = [];
         progress(count, total, { skipped, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) { errors++; count++; }
   }

package/marketplace.js

@@ -61,6 +61,16 @@ function validateRegistryEntry(item) {
   }
   if (item.downloads !== undefined && typeof item.downloads !== 'number') errors.push('downloads must be a number');
   if (item.verified !== undefined && typeof item.verified !== 'boolean') errors.push('verified must be a boolean');
+  if (item.trustLevel !== undefined && typeof item.trustLevel !== 'string') errors.push('trustLevel must be a string');
+  if (item.rating !== undefined) {
+    if (typeof item.rating !== 'number') errors.push('rating must be a number');
+    else if (item.rating < 0 || item.rating > 5) errors.push('rating must be between 0 and 5');
+  }
+  if (item.popularity !== undefined && typeof item.popularity !== 'number') errors.push('popularity must be a number');
+  if (item.lastUpdate !== undefined) {
+    if (typeof item.lastUpdate !== 'string') errors.push('lastUpdate must be a string');
+    else if (isNaN(Date.parse(item.lastUpdate))) errors.push(`lastUpdate "${item.lastUpdate}" is not a valid ISO date`);
+  }
   if (errors.length > 0) {
     console.warn(`[PromptGraph] Skipping invalid registry entry "${item.id || item.name || '(unnamed)'}": ${errors.join('; ')}`);
     return { ok: false };
@@ -162,7 +172,7 @@ export async function browseMarketplace(topK = 20) {
     const registry = JSON.parse(text);
     if (!Array.isArray(registry.skills)) return { error: 'Invalid registry format' };
     return registry.skills
-      .map(s => ({ ...s, code: s.code || codeFor(s.id) })) // auto-fill code
+      .map(s => Object.assign(Object.create(null), s, { code: s.code || codeFor(s.id) }))
       .filter(s => validateRegistryEntry(s).ok)
       .filter(s => s.raw_url)
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
@@ -287,7 +297,7 @@ export async function browseBundles(topK = 20) {
 
     return bundles
       .filter(b => !b.repo_url || b.skillCount > 0)
-      .map(b => ({ ...b, code: b.code || codeFor(b.id) }))
+      .map(b => Object.assign(Object.create(null), b, { code: b.code || codeFor(b.id) }))
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
   } catch (e) {
@@ -548,3 +558,74 @@ export function pruneInvalidRepos() {
   saveConfig(config);
   return { removed, kept, errors };
 }
+
+// ── Trust level system ─────────────────────────────────────────────────────────
+const VALID_TRUST_LEVELS = ['verified', 'official', 'community', 'trusted', 'unknown']
+
+function calcPopularity(downloads = 0, rating = 0) {
+  return Math.round((downloads || 0) * ((rating || 0) + 1) * 100) / 100
+}
+
+export async function setTrustLevel(name, level) {
+  const db = getDb()
+  if (!VALID_TRUST_LEVELS.includes(level)) {
+    return { ok: false, error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  const id = String(name)
+  db.prepare(`
+    INSERT INTO registry_entries (id, trust_level, last_update)
+    VALUES (?, ?, datetime('now'))
+    ON CONFLICT(id) DO UPDATE SET trust_level = excluded.trust_level, last_update = excluded.last_update
+  `).run(id, level)
+  return { ok: true }
+}
+
+export async function getByTrustLevel(level) {
+  const db = getDb()
+  if (level && !VALID_TRUST_LEVELS.includes(level)) {
+    return { error: `Invalid trust level "${level}". Must be one of: ${VALID_TRUST_LEVELS.join(', ')}` }
+  }
+  if (level) {
+    return db.prepare('SELECT * FROM registry_entries WHERE trust_level = ? ORDER BY id').all(level)
+  }
+  return db.prepare('SELECT * FROM registry_entries ORDER BY id').all()
+}
+
+export async function incrementDownloads(name) {
+  const db = getDb()
+  const id = String(name)
+  const existing = db.prepare('SELECT downloads, rating FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const newDownloads = (existing.downloads || 0) + 1
+    const pop = calcPopularity(newDownloads, existing.rating || 0)
+    db.prepare('UPDATE registry_entries SET downloads = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newDownloads, pop, id)
+  } else {
+    const pop = calcPopularity(1, 0)
+    db.prepare('INSERT INTO registry_entries (id, downloads, popularity, last_update) VALUES (?, 1, ?, datetime(\'now\'))')
+      .run(id, pop)
+  }
+  return { ok: true }
+}
+
+export async function rateSkill(name, rating) {
+  const db = getDb()
+  if (typeof rating !== 'number' || rating < 0 || rating > 5) {
+    return { ok: false, error: 'Rating must be a number between 0 and 5' }
+  }
+  const id = String(name)
+  const existing = db.prepare('SELECT rating, rating_count, downloads FROM registry_entries WHERE id = ?').get(id)
+  if (existing) {
+    const count = existing.rating_count || 0
+    const newRating = Math.round(((existing.rating || 0) * count + rating) / (count + 1) * 100) / 100
+    const newCount = count + 1
+    const pop = calcPopularity(existing.downloads || 0, newRating)
+    db.prepare('UPDATE registry_entries SET rating = ?, rating_count = ?, popularity = ?, last_update = datetime(\'now\') WHERE id = ?')
+      .run(newRating, newCount, pop, id)
+  } else {
+    const pop = calcPopularity(0, rating)
+    db.prepare('INSERT INTO registry_entries (id, rating, rating_count, popularity, last_update) VALUES (?, ?, 1, ?, datetime(\'now\'))')
+      .run(id, rating, pop)
+  }
+  return { ok: true }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.8",
+  "version": "2.5.0",
   "files": [
     "*.js",
     "src/**/*.js",
@@ -55,6 +55,6 @@
     "vitest": "^4.1.8"
   },
   "overrides": {
-    "tar": "^6.2.1"
+    "tar": "^7.5.11"
   }
 }

package/search.js

@@ -2,24 +2,10 @@ import { embed, cosineSimilarity } from './embedder.js';
 import { getDb, blobToVec } from './db.js';
 import { annSearch } from './ann.js';
 
-let embedWeight = 0.7;
-let bm25Weight = 0.3;
-
-export function setHybridWeights(ew, bw) {
-  embedWeight = ew;
-  bm25Weight = bw;
-}
-
-function adaptWeights(query) {
-  // Technical terms (caps, digits) → BM25 matters more
+function getHybridWeights(query) {
   const hasTechTerms = /[A-Z]|\d/.test(query);
-  if (hasTechTerms) {
-    embedWeight = 0.5;
-    bm25Weight = 0.5;
-  } else {
-    embedWeight = 0.7;
-    bm25Weight = 0.3;
-  }
+  if (hasTechTerms) return { embedWeight: 0.5, bm25Weight: 0.5 }
+  return { embedWeight: 0.7, bm25Weight: 0.3 }
 }
 
 function applyRatingBoost(db, id, score) {
@@ -82,7 +68,7 @@ function runBM25Search(db, query, topK) {
 
 export async function search(query, topK = 5) {
   const db = getDb();
-  adaptWeights(query);
+  const { embedWeight, bm25Weight } = getHybridWeights(query);
   const queryVec = await embed(query);
 
   const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
@@ -116,13 +102,29 @@ export async function search(query, topK = 5) {
     }
   }
 
-  return [...combined.entries()]
+  const ordered = [...combined.entries()]
     .map(([id, s]) => ({
       id,
       score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
     }))
     .sort((a, b) => b.score - a.score)
-    .slice(0, topK)
+
+  const rerankerEnabled = process.env.PG_RERANKER !== '0'
+
+  if (rerankerEnabled) {
+    const { Reranker } = await import('./src/reranker/reranker.js')
+    const reranker = new Reranker()
+    const topN = ordered.slice(0, 20)
+      .map(({ id, score }) => {
+        const s = skillWithSnippet(db, id, score)
+        return s ? { id, text: s.snippet, score } : null
+      })
+      .filter(Boolean)
+    const reranked = await reranker.rerank(query, topN)
+    return reranked.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score))).filter(Boolean)
+  }
+
+  return ordered.slice(0, topK)
     .map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
     .filter(Boolean);
 }

package/src/reranker/reranker.js

@@ -0,0 +1,27 @@
+export class Reranker {
+  constructor(modelName = 'default', device = 'cpu') {
+    this.modelName = modelName
+    this.device = device
+  }
+
+  // Cross-encoder style reranker
+  // Plug-in point for BGE Reranker / MiniLM Reranker via ONNX
+  async rerank(query, results) {
+    if (!results || results.length === 0) return []
+
+    const queryTerms = (query.toLowerCase().match(/\w+/g) || [])
+    const queryTermCount = queryTerms.length || 1
+
+    const scored = results.map(r => {
+      const text = (r.text || '').toLowerCase()
+      const termOverlap = queryTerms.filter(t => text.includes(t)).length / queryTermCount
+
+      return {
+        ...r,
+        score: 0.8 * (r.score || 0) + 0.2 * termOverlap,
+      }
+    })
+
+    return scored.sort((a, b) => b.score - a.score).slice(0, 5)
+  }
+}

package/src/utils/rate-limiter.js

@@ -0,0 +1,33 @@
+export class RateLimiter {
+  constructor({ maxRequests, windowMs }) {
+    this.maxRequests = maxRequests
+    this.windowMs = windowMs
+    this.timestamps = []
+    this._chain = Promise.resolve()
+  }
+
+  async acquire() {
+    const prev = this._chain
+    this._chain = prev.then(() => this._doAcquire())
+    return this._chain
+  }
+
+  async _doAcquire() {
+    while (!this.tryAcquire()) {
+      const oldest = this.timestamps[0]
+      const waitMs = oldest
+        ? Math.max(1, oldest + this.windowMs - Date.now())
+        : 100
+      await new Promise(r => setTimeout(r, Math.min(waitMs, this.windowMs)))
+    }
+  }
+
+  tryAcquire() {
+    const now = Date.now()
+    const cutoff = now - this.windowMs
+    this.timestamps = this.timestamps.filter(t => t > cutoff)
+    if (this.timestamps.length >= this.maxRequests) return false
+    this.timestamps.push(now)
+    return true
+  }
+}

package/validator.js

@@ -1,5 +1,7 @@
 import fs from 'fs';
+import path from 'path';
 import matter from 'gray-matter';
+import { MAX_DOWNLOAD_SIZE } from './config.js';
 
 // patterns that indicate malicious or junk skills
 const DANGEROUS_PATTERNS = [
@@ -86,6 +88,19 @@ export function validateSkill(filePath) {
   if (pathParts.includes('..')) {
     errors.push('Path traversal detected: file path contains ".."');
   }
+
+  // extension whitelist — reject unexpected file types
+  const ALLOWED_EXTENSIONS = new Set(['.md', '.json', '.yaml', '.yml', '.txt', '.js', '.ts', '.py', '.rs', '.toml']);
+  const ext = path.extname(filePath).toLowerCase();
+  if (!ALLOWED_EXTENSIONS.has(ext)) {
+    errors.push(`File extension "${ext}" not allowed. Allowed: ${[...ALLOWED_EXTENSIONS].join(', ')}`);
+  }
+
+  // binary content detection — must be valid UTF-8 with no null bytes
+  if (raw.includes('\0')) {
+    errors.push('File contains null bytes (binary content)');
+  }
+
   if (['readme.md', 'changelog.md', 'license.md', 'contributing.md'].includes(base)) {
     warnings.push('Filename looks like a docs file, not a skill.');
   }
@@ -142,6 +157,15 @@ export function validateBundle(def) {
   return { ok: errors.length === 0, errors, warnings };
 }
 
+export function sanitizeExternalContent(content) {
+  if (typeof content !== 'string') return ''
+  let sanitized = content.replace(/\0/g, '')
+  if (Buffer.byteLength(sanitized, 'utf8') > MAX_DOWNLOAD_SIZE) {
+    sanitized = Buffer.from(sanitized, 'utf8').subarray(0, MAX_DOWNLOAD_SIZE).toString('utf8')
+  }
+  return sanitized
+}
+
 // CLI: node validator.js <file>
 if (import.meta.url === `file://${process.argv[1]}`) {
   const file = process.argv[2];

package/src/filter/cluster.js

@@ -1,52 +0,0 @@
-import { cosineSimilarity } from '../../embedder.js';
-
-const INFERENCE_THRESHOLD = 0.35;
-
-export function findClusters(skills, embeddings) {
-  if (!skills.length || !embeddings) return [];
-  const clusters = [];
-  const assigned = new Set();
-
-  for (let i = 0; i < skills.length; i++) {
-    if (assigned.has(i)) continue;
-    const cluster = { centroid: embeddings[i], members: [skills[i]], indices: [i] };
-    assigned.add(i);
-
-    for (let j = i + 1; j < skills.length; j++) {
-      if (assigned.has(j)) continue;
-      if (cosineSimilarity(embeddings[i], embeddings[j]) >= INFERENCE_THRESHOLD) {
-        cluster.members.push(skills[j]);
-        cluster.indices.push(j);
-        assigned.add(j);
-      }
-    }
-
-    clusters.push(cluster);
-  }
-
-  return clusters;
-}
-
-export function expandResults(annResults, db) {
-  if (!annResults || annResults.length === 0) return annResults;
-
-  const seenIds = new Set();
-  const expanded = [];
-
-  for (const r of annResults) {
-    if (!seenIds.has(r.skill_id)) {
-      seenIds.add(r.skill_id);
-      expanded.push(r);
-    }
-
-    const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(r.skill_id);
-    for (const c of callees) {
-      if (!seenIds.has(c.to_skill)) {
-        seenIds.add(c.to_skill);
-        expanded.push({ skill_id: c.to_skill, score: r.score * 0.85 });
-      }
-    }
-  }
-
-  return expanded.sort((a, b) => b.score - a.score);
-}

package/src/filter/dedup.js

@@ -1,36 +0,0 @@
-import { cosineSimilarity } from '../../embedder.js';
-
-const DEDUP_THRESHOLD = 0.97;
-
-export function dedup(skills, embeddings) {
-  if (!skills.length) return { skills: [], embeddings: [] };
-  if (embeddings && embeddings.length !== skills.length) {
-    throw new Error('embeddings length must match skills length');
-  }
-
-  const keep = [true]; // first skill always kept
-  const kept = [0];
-
-  for (let i = 1; i < skills.length; i++) {
-    let dup = false;
-    if (embeddings) {
-      for (const j of kept) {
-        if (cosineSimilarity(embeddings[i], embeddings[j]) >= DEDUP_THRESHOLD) {
-          dup = true;
-          break;
-        }
-      }
-    }
-    if (dup) {
-      keep.push(false);
-    } else {
-      keep.push(true);
-      kept.push(i);
-    }
-  }
-
-  return {
-    skills: skills.filter((_, i) => keep[i]),
-    embeddings: embeddings ? embeddings.filter((_, i) => keep[i]) : undefined,
-  };
-}