promptgraph-mcp 2.6.3 → 2.6.5

package/commands/marketplace.js

@@ -5,6 +5,22 @@ import os from 'os';
 import fs from 'fs';
 
 export default async function handler(args, bin) {
+  // Subcommand: validate / prune all installed marketplace files
+  if (args[0] === 'validate' || args[0] === 'prune' || args[0] === '--validate' || args[0] === '--prune') {
+    const { validateAndPruneMarketplace } = await import('../marketplace.js');
+    const result = validateAndPruneMarketplace();
+    if (result.removed.length > 0) {
+      error(`Removed ${result.removed.length} invalid files:`);
+      result.removed.forEach(r => console.log(`  ${chalk.red('✗')} ${r.file}`));
+    }
+    if (result.errors.length > 0) {
+      error(`${result.errors.length} errors:`);
+      result.errors.forEach(e => console.log(`  ${chalk.yellow('⚠')} ${e}`));
+    }
+    success(`${result.valid.length} valid files, ${result.removed.length} removed, ${result.errors.length} errors`);
+    process.exit(result.errors.length > 0 ? 1 : 0);
+  }
+
   if (!process.stdout.isTTY) {
     error('marketplace TUI requires an interactive terminal');
     process.exit(1);
@@ -62,6 +78,8 @@ export default async function handler(args, bin) {
   const { loadConfig: _lcR, saveConfig: _scR, SKILLS_STORE_DIR: _ssR } = await import('../config.js');
   const { getDb: _getDbR } = await import('../db.js');
 
+  const { validateAndPruneMarketplace } = await import('../marketplace.js');
+
   await runTUI(
     Array.isArray(skills) ? skills : [],
     bundlesWithCounts,
@@ -76,6 +94,11 @@ export default async function handler(args, bin) {
         installedSet.add(item.id);
         if (item.code) installedSet.add(item.code);
       }
+      // After every install, prune invalid files marketplace-wide
+      const pruneResult = validateAndPruneMarketplace();
+      if (pruneResult.removed.length > 0) {
+        console.log(`Pruned ${pruneResult.removed.length} invalid files from marketplace.`);
+      }
     },
     installedSet,
     async (item) => {

package/github-import.js

@@ -6,7 +6,7 @@ import { globSync } from 'glob';
 import { indexAll, indexSource } from './indexer.js';
 import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR, MAX_DOWNLOAD_SIZE, MAX_FILE_COUNT, MAX_REPO_SIZE, RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_MS } from './config.js';
 import { validateSkill } from './validator.js';
-import { isSkillFile } from './parser.js';
+import { isSkillFile, filterWithClassifier, parseSkillFile } from './parser.js';
 import { RateLimiter } from './src/utils/rate-limiter.js';
 
 const githubRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS, windowMs: RATE_LIMIT_WINDOW_MS })
@@ -169,7 +169,7 @@ export async function validateRepoSkills(ownerRepo) {
   }
 
   // Filter out docs-like filenames (README, LICENSE, CHANGELOG, etc.)
-  const SKIP_DOCS = /^(readme|license|changelog|contributing|code.?of.?conduct|security|authors|credits|install|faq|index|overview|summary|todo|notes|template|copying|warranty|funding|roadmap)/i;
+  const SKIP_DOCS = /^(readme|license|changelog|contributing|code.?of.?conduct|security|authors|credits|install|faq|index|overview|summary|todo|notes|template|copying|warranty|funding|roadmap|claude|bugs?\b|feature.?request)/i;
   const mdTrimmed = mdFiles.filter(f => !SKIP_DOCS.test(f.name.replace(/\.md$/i, '')));
   const mdToValidate = mdTrimmed.length > 0 ? mdTrimmed : mdFiles;
 
@@ -238,6 +238,7 @@ const SKIP_DIRS_API = new Set([
   'node_modules', 'vendor', 'dist', 'build', '.git',
   'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
   'cheatsheets', 'resources',
+  'src', 'cli', 'lib', 'bin', 'scripts',
 ]);
 
 function git(args, cwd, stdio = 'inherit') {
@@ -269,13 +270,14 @@ function sparseClone(url, dest, subdir) {
 }
 
 // Shared skip patterns — module scope so both cleanup functions can access them
-const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
+const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding|claude|bugs?\b|feature.?request)/i;
 const SKIP_DIRS_LOCAL = new Set([
   '.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots',
   'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor',
   'dist', 'build', 'tests', 'test',
   'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
   'cheatsheets', 'resources',
+  'src', 'cli', 'lib', 'bin',
 ]);
 
 // After full-clone root: remove files that are not skills and dirs we don't need
@@ -427,6 +429,54 @@ function detectSkillsDirLocal(repoRoot) {
   return { dir: repoRoot, label: '(root)', sparse: false };
 }
 
+// Remove files classified as non-skills by embedding classifier (only if model is trained)
+async function classifierCleanup(dest) {
+  const mdFiles = globSync(`${dest}/**/*.md`);
+  if (mdFiles.length === 0) return;
+
+  const parsed = [];
+  const fileMap = [];
+  let heuristicRemoved = 0;
+
+  for (const fp of mdFiles) {
+    try {
+      const raw = fs.readFileSync(fp, 'utf8');
+      if (!isSkillFile(fp, raw)) continue;
+      if (!seemsLikeSkill(raw)) {
+        try { fs.unlinkSync(fp); heuristicRemoved++; } catch {}
+        continue;
+      }
+      parsed.push(parseSkillFile(fp, '', { raw }));
+      fileMap.push(fp);
+    } catch {}
+  }
+
+  if (heuristicRemoved > 0) {
+    console.log(`Removed ${heuristicRemoved} files by content heuristic`);
+  }
+
+  if (parsed.length === 0) {
+    if (heuristicRemoved > 0) removeEmptyDirs(dest);
+    return;
+  }
+
+  const filtered = await filterWithClassifier(parsed);
+  const keptPaths = new Set(filtered.map(s => s.path));
+
+  let classifierRemoved = 0;
+  for (const fp of fileMap) {
+    if (!keptPaths.has(fp)) {
+      try { fs.unlinkSync(fp); classifierRemoved++; } catch {}
+    }
+  }
+
+  const totalRemoved = heuristicRemoved + classifierRemoved;
+  if (totalRemoved > 0) {
+    console.log(`Removed ${totalRemoved} non-skill files (heuristic: ${heuristicRemoved}, classifier: ${classifierRemoved})`);
+    removeEmptyDirs(dest);
+  }
+}
+
 // ── main export ───────────────────────────────────────────────────────────────
 
 export async function importFromGitHub(repoUrl) {
@@ -499,6 +549,7 @@ export async function importFromGitHub(repoUrl) {
 
   // Remove doc files anywhere in the cloned tree
   cleanupRepoRoot(dest);
+  removeEmptyDirs(dest);
 
   // Validate every .md file via isSkillFile — delete low-quality files
   const allMd = globSync(`${dest}/**/*.md`);
@@ -510,10 +561,37 @@ export async function importFromGitHub(repoUrl) {
   }
   if (removedInvalid > 0) {
     console.log(`Removed ${removedInvalid} low-quality .md files (isSkillFile)`);
-    // Clean up empty dirs left behind
     removeEmptyDirs(dest);
   }
 
+  // Full validateSkill() pass — remove files that fail marketplace-level validation
+  const remainingMd = globSync(`${dest}/**/*.md`);
+  let removedFailedValidation = 0;
+  for (const fp of remainingMd) {
+    const v = validateSkill(fp);
+    if (!v.ok) {
+      try { fs.unlinkSync(fp); removedFailedValidation++; } catch {}
+    }
+  }
+  if (removedFailedValidation > 0) {
+    console.log(`Removed ${removedFailedValidation} files that failed validateSkill()`);
+    removeEmptyDirs(dest);
+  }
+
+  await classifierCleanup(dest);
+
+  // Update skill count cache with real survivor count
+  const realCount = globSync(`${dest}/**/*.md`).length;
+  const cachePath = path.join(PROMPTGRAPH_DIR, 'skill-counts.json');
+  try {
+    const cache = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+    if (cache[url]) {
+      cache[url].count = realCount;
+      cache[url].ts = Date.now();
+      fs.writeFileSync(cachePath, JSON.stringify(cache, null, 2));
+    }
+  } catch {}
+
   const { dir: localDir, label: localLabel } = detectSkillsDirLocal(dest);
   // Prefer the known skillsSubdir (from API detection or sparse patterns) as the
   // canonical skills directory — it's more accurate than local heuristics for

package/marketplace.js

@@ -9,6 +9,7 @@ import { globSync } from 'glob';
 import { validateSkill, validateBundle } from './validator.js';
 import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './config.js';
 import { importFromGitHub, validateRepoSkills } from './github-import.js';
+import { isSkillFile } from './parser.js';
 
 const REGISTRY_URL = 'https://raw.githubusercontent.com/NeiP4n/promptgraph-registry/main/registry.json';
 const SKILL_COUNT_CACHE = path.join(PROMPTGRAPH_DIR, 'skill-counts.json');
@@ -248,12 +249,8 @@ export async function installSkill(query) {
   }
 }
 
-// ── filter skill files (exclude docs) ─────────────────────────────────────────
-const SKIP_DOCS = /^(readme|license|changelog|contributing|code.?of.?conduct|security|authors|credits|install|faq|index|overview|summary|todo|notes|template|copying|warranty|funding|roadmap)/i;
-function isSkillFile(path) {
-  const name = path.split('/').pop().toLowerCase();
-  return name.endsWith('.md') && !SKIP_DOCS.test(name.replace(/\.md$/i, ''));
-}
+// ── filter skill files (exclude docs) — delegates to parser.js isSkillFile ─────
+// was: local isSkillFile(path) — removed in favor of shared parser.js version
 
 async function countRepoSkills(repoUrl) {
   try {
@@ -643,6 +640,55 @@ export async function incrementDownloads(name) {
   return { ok: true }
 }
 
+// ── validate & prune all installed marketplace files ──────────────────────────
+
+export function validateAndPruneMarketplace() {
+  const results = { valid: [], removed: [], errors: [] };
+  if (!fs.existsSync(SKILLS_DIR)) {
+    return { ...results, message: 'No marketplace directory found.' };
+  }
+
+  const mdFiles = globSync(`${SKILLS_DIR}/**/*.md`, { absolute: true });
+  for (const fp of mdFiles) {
+    const name = path.relative(SKILLS_DIR, fp);
+    try {
+      const v = validateSkill(fp);
+      if (!v.ok) {
+        try { fs.unlinkSync(fp); results.removed.push({ file: name, errors: v.errors }); } catch (e) { results.errors.push(`Failed to remove ${name}: ${e.message}`); }
+      } else {
+        results.valid.push(name);
+      }
+    } catch (e) {
+      results.errors.push(`Error validating ${name}: ${e.message}`);
+    }
+  }
+
+  // Clean up empty dirs left behind
+  removeEmptyDirs(SKILLS_DIR);
+
+  // Also remove DB entries for deleted files
+  const db = getDb();
+  for (const row of db.prepare('SELECT id, path FROM skills WHERE source = ?').all('marketplace')) {
+    if (!fs.existsSync(row.path)) {
+      db.prepare('DELETE FROM skills WHERE id = ?').run(row.id);
+      db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(row.id);
+    }
+  }
+
+  return results;
+}
+
+function removeEmptyDirs(dirPath) {
+  let entries;
+  try { entries = fs.readdirSync(dirPath, { withFileTypes: true }); } catch { return; }
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const fullPath = path.join(dirPath, entry.name);
+    removeEmptyDirs(fullPath);
+    try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
+  }
+}
+
 export { VALID_TRUST_LEVELS, TRUST_LEVEL_BOOST, calcPopularity, autoPromote }
 
 export async function rateSkill(name, rating) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.6.3",
+  "version": "2.6.5",
   "files": [
     "*.js",
     "commands/",

package/src/filter/hard-filter.js

@@ -10,9 +10,11 @@ const SKIP_FILENAMES = new Set([
   'design', 'spec', 'specification', 'requirements', 'privacy', 'terms',
   'disclaimer', 'notice', 'copying', 'warranty', 'codeofconduct',
   'pull_request_template', 'issue_template', 'funding',
+  'claude', 'bugs', 'bug_report', 'bug-report', 'feature_request',
+  'feature-request',
 ]);
 
-const SKIP_FILENAME_RE = /^(_|\.)|^v?\d+[\.\-]\d+|^\d{4}[\-_]\d{2}|^readme|^license|^changelog|^contributing|^code.of.conduct|^security|^authors|^credits|^disclaimer|^notice|^copying|^warranty|^promotion|^funding/i;
+const SKIP_FILENAME_RE = /^(_|\.)|^v?\d+[\.\-]\d+|^\d{4}[\-_]\d{2}|^readme|^license|^changelog|^contributing|^code.of.conduct|^security|^authors|^credits|^disclaimer|^notice|^copying|^warranty|^promotion|^funding|^claude|^bugs?\b|^feature.?request/i;
 
 const SKIP_DIRS = new Set([
   '.github', 'docs', 'doc', 'documentation', 'examples', 'example',
@@ -21,6 +23,7 @@ const SKIP_DIRS = new Set([
   'node_modules', 'vendor', 'third_party',
   'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
   'cheatsheets', 'resources',
+  'src', 'cli', 'lib', 'bin',
 ]);
 
 const BADGE_RE = /!\[.*\]\(https?:\/\/(img\.shields\.io|badge\.fury|travis-ci|github\.com\/[^)]+\/badge)/i;