promptgraph-mcp 2.4.6 → 2.4.7

package/github-import.js

@@ -6,6 +6,7 @@ import { globSync } from 'glob';
 import { indexAll, indexSource } from './indexer.js';
 import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './config.js';
 import { validateSkill } from './validator.js';
+import { isSkillFile } from './parser.js';
 
 const SKILL_DIRS = ['skills', 'commands', 'prompts', 'agents', 'skills-store', 'slash-commands', 'custom-commands', 'templates'];
 
@@ -366,17 +367,16 @@ export async function importFromGitHub(repoUrl) {
   // Remove doc files anywhere in the cloned tree
   cleanupRepoRoot(dest);
 
-  // Validate every .md file — delete those that fail
+  // Validate every .md file via isSkillFile — delete low-quality files
   const allMd = globSync(`${dest}/**/*.md`);
   let removedInvalid = 0;
   for (const fp of allMd) {
-    const v = validateSkill(fp);
-    if (!v.ok) {
+    if (!isSkillFile(fp)) {
       try { fs.unlinkSync(fp); removedInvalid++; } catch {}
     }
   }
   if (removedInvalid > 0) {
-    console.log(`Removed ${removedInvalid} invalid .md files (failed validation)`);
+    console.log(`Removed ${removedInvalid} low-quality .md files (isSkillFile)`);
     // Clean up empty dirs left behind
     removeEmptyDirs(dest);
   }

package/index.js

@@ -16,7 +16,7 @@ const args = process.argv.slice(2);
 const rawBin = process.argv[1]?.split(/[\\/]/).pop()?.replace(/\.js$/, '');
 const bin = (rawBin && rawBin !== 'index') ? rawBin : 'pg';
 
-const KNOWN_COMMANDS = new Set(['init', 'reindex', 'update', 'import', 'setup', 'validate', 'marketplace', 'doctor', 'search', 'help', '--help', '-h', 'bundle', 'status']);
+const KNOWN_COMMANDS = new Set(['init', 'reindex', 'update', 'import', 'setup', 'validate', 'marketplace', 'doctor', 'search', 'help', '--help', '-h', 'bundle', 'status', 'train']);
 
 function showHelp() {
   console.log(
@@ -319,24 +319,41 @@ if (args[0] === 'validate') {
 
   const raw = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null;
 
-  // Show indexing score breakdown
   if (raw) {
-    const { skillScore: _score } = await import('./parser.js').catch(() => ({}));
-    const willIndex = isSkillFile(file, raw);
+    const { filterWithClassifier, isSkillFile: _isSkill } = await import('./parser.js');
+    const { hardFilter } = await import('./src/filter/hard-filter.js');
+    const { loadModel } = await import('./src/filter/train.js');
+    const { embed } = await import('./embedder.js');
+    const { classify } = await import('./src/filter/classifier.js');
+
+    const hfResult = hardFilter(file, raw);
+    const willIndex = _isSkill(file, raw);
     const scoreLabel = willIndex ? chalk.green('✓ will be indexed') : chalk.red('✗ will be skipped by indexer');
     console.log(chalk.bold('\n  Indexing check: ') + scoreLabel);
 
-    // Show which signals were detected
-    const lines = raw.split('\n').filter(l => l.trim());
     const signals = [];
-    try { const { data } = (await import('gray-matter')).default(raw); if (data.name) signals.push(chalk.green('+4 frontmatter name:')); } catch {}
-    if (/^#{1,3}\s+(steps?|usage|instructions?|how\s+to|when\s+to\s+use|workflow)/im.test(raw)) signals.push(chalk.green('+2 instructional headers (## Steps / ## Usage)'));
-    if (lines.filter(l => /^#{1,3}\s/.test(l)).some(h => /\b(run|use|fix|debug|check|create|deploy|scan|audit)\b/i.test(h))) signals.push(chalk.green('+2 imperative verbs in headers'));
-    if (raw.includes('```')) signals.push(chalk.green('+1 code block'));
-    if (lines.some(l => /^\d+\.\s/.test(l))) signals.push(chalk.green('+1 numbered list'));
-    if (lines.some(l => /^[-*+]\s/.test(l))) signals.push(chalk.green('+1 bullet list'));
-    const firstH = lines.find(l => /^#{1,3}\s/.test(l))?.replace(/^#+\s*/, '') || '';
-    if (/^(overview|introduction|about|background|welcome)/i.test(firstH)) signals.push(chalk.red('-3 first header looks like docs ("' + firstH + '")'));
+    if (!hfResult.pass) {
+      signals.push(chalk.red(`✗ hard filter: ${hfResult.reason}`));
+    } else {
+      signals.push(chalk.green('✓ hard filter passed'));
+    }
+
+    const centroids = loadModel();
+    if (centroids) {
+      try {
+        const vec = await embed(raw);
+        const decision = classify(vec, centroids, raw, file);
+        const pct = (decision.score * 100).toFixed(0);
+        if (decision.label === 'skill') signals.push(chalk.green(`✓ classifier: skill (${pct}%)`));
+        else if (decision.label === 'unsure') signals.push(chalk.yellow(`? classifier: unsure (${pct}%)`));
+        else signals.push(chalk.red(`✗ classifier: reject (${pct}%)`));
+      } catch {
+        signals.push(chalk.gray('  classifier: embed failed (skip)'));
+      }
+    } else {
+      signals.push(chalk.gray('  classifier: no model (run `pg train`)'));
+    }
+
     if (signals.length) {
       signals.forEach(s => console.log('    ' + s));
     }
@@ -355,6 +372,22 @@ if (args[0] === 'validate') {
   }
 }
 
+if (args[0] === 'train') {
+  const { train: trainModel } = await import('./src/filter/train.js');
+  const spin = (await import('./cli.js')).spinner('Training classifier...');
+  spin.start();
+  try {
+    const model = await trainModel();
+    spin.stop();
+    success(`Classifier trained (${model.counts.good} good, ${model.counts.bad} bad examples)`);
+  } catch (e) {
+    spin.stop();
+    error(`Training failed: ${e.message}`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
 if (args[0] === 'search') {
   const query = args.slice(1).join(' ');
   if (!query) { error('Usage: ' + bin + ' search <query>'); process.exit(1); }

package/indexer.js

@@ -2,8 +2,8 @@ import { globSync } from 'glob';
 import { createHash } from 'crypto';
 import fs from 'fs';
 import path from 'path';
-import { parseSkillFile, isSkillFile } from './parser.js';
-import { embedBatch, BATCH_SIZE } from './embedder.js';
+import { parseSkillFile, isSkillFile, filterWithClassifier } from './parser.js';
+import { embedBatch, cosineSimilarity, BATCH_SIZE } from './embedder.js';
 import { getDb, skillId, vecToBlob } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
@@ -138,6 +138,7 @@ export async function indexAll({ fast = false } = {}) {
   let count = 0;
   let errors = 0;
   let skipped = 0;
+  let classifierRemoved = 0;
   let batch = [];
   const start = Date.now();
 
@@ -174,8 +175,10 @@ export async function indexAll({ fast = false } = {}) {
       batch.push({ ...parsed, hash });
 
       if (batch.length >= BATCH_SIZE) {
-        await indexBatch(db, batch, { fast });
-        count += batch.length;
+        const filtered = await filterWithClassifier(batch);
+        classifierRemoved += batch.length - filtered.length;
+        await indexBatch(db, filtered, { fast });
+        count += filtered.length;
         batch = [];
         const eta = count > 0 ? Math.round((total - count) * (Date.now() - start) / count / 1000) : '?';
         progress(count, total, { skipped, eta, errors });
@@ -196,8 +199,10 @@ export async function indexAll({ fast = false } = {}) {
   }
 
   if (batch.length > 0) {
-    await indexBatch(db, batch, { fast });
-    count += batch.length;
+    const filtered = await filterWithClassifier(batch);
+    classifierRemoved += batch.length - filtered.length;
+    await indexBatch(db, filtered, { fast });
+    count += filtered.length;
   }
 
   progress(total, total, { skipped, errors });
@@ -208,7 +213,9 @@ export async function indexAll({ fast = false } = {}) {
     await buildAnnIndex();
     spin.stop();
   }
-  success(`Indexed ${chalk.white.bold(count)} skills  ${chalk.gray(`(${errors} errors, ${skipped} skipped, ${removed} removed)`)}`);
+  const stats = [`${errors} errors`, `${skipped} skipped`, `${removed} removed`];
+  if (classifierRemoved > 0) stats.push(`${classifierRemoved} filtered`);
+  success(`Indexed ${chalk.white.bold(count)} skills  ${chalk.gray(`(${stats.join(', ')})`)}`);
   if (fast) info(chalk.yellow('Fast mode: keyword search only. Run `pg reindex` for semantic search.'));
   const elapsed = ((Date.now() - start) / 1000).toFixed(1);
   info(chalk.gray(`Time: ${elapsed}s`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.6",
+  "version": "2.4.7",
   "main": "index.js",
   "type": "module",
   "bin": {

package/parser.js

@@ -1,136 +1,42 @@
 import fs from 'fs';
 import path from 'path';
 import matter from 'gray-matter';
+import { hardFilter } from './src/filter/hard-filter.js';
+import { classify } from './src/filter/classifier.js';
+import { loadModel } from './src/filter/train.js';
 
-// match /skill-name but not URLs (http://, https://, etc.)
 const SKILL_REF_RE = /(?<!https?:|ftp:)(?<![a-zA-Z0-9])\/([a-z0-9][a-z0-9-]{2,})/g;
 
-// Filenames that are never skills — docs, meta, legal files
-const SKIP_FILENAMES = new Set([
-  'readme', 'changelog', 'license', 'contributing', 'code-of-conduct',
-  'security', 'authors', 'credits', 'install', 'installation', 'usage',
-  'engagements', 'contributors', 'maintainers', 'acknowledgements',
-  'faq', 'glossary', 'index', 'overview', 'summary', 'roadmap', 'todo',
-  'notes', 'template', 'example', 'sample', 'demo', 'getting-started',
-  'quickstart', 'guide', 'tutorial', 'walkthrough', 'architecture',
-  'design', 'spec', 'specification', 'requirements', 'privacy', 'terms',
-  'disclaimer', 'notice', 'copying', 'warranty', 'codeofconduct',
-  'pull_request_template', 'issue_template', 'funding',
-]);
-
-// Filename patterns that are never skills — readme* catches ALL variants (readme_de, readme_zh-CN, etc.)
-const SKIP_FILENAME_RE = /^(_|\.)|^v?\d+[\.\-]\d+|^\d{4}[\-_]\d{2}|^readme|^license|^changelog|^contributing|^code.of.conduct|^security|^authors|^credits|^disclaimer|^notice|^copying|^warranty|^promotion|^funding/i;
-
-// Path segments that indicate the file is NOT a skill
-const SKIP_DIRS = new Set([
-  '.github', 'docs', 'doc', 'documentation', 'examples', 'example',
-  'tests', 'test', '__tests__', 'spec', 'fixtures', 'assets', 'images',
-  'img', 'screenshots', 'media', 'static', 'public', 'dist', 'build',
-  'node_modules', 'vendor', 'third_party',
-]);
-
-// First-header values that signal documentation, not a skill
-const DOC_FIRST_HEADERS = /^(overview|introduction|about|background|welcome|getting started|what is|why |table of contents|toc|foreword|preface|readme)/i;
-
-// Imperative verbs commonly found in skill headers
-const IMPERATIVE_HEADERS = /\b(run|use|apply|execute|check|debug|fix|create|add|remove|deploy|test|write|generate|analyze|review|refactor|optimize|configure|setup|install|scan|audit|validate|search|find|extract|parse)\b/i;
-
-// Instructional section headers
-const INSTRUCTION_HEADERS = /^#{1,3}\s+(steps?|usage|instructions?|how\s+to|when\s+to\s+use|workflow|process|procedure|example|examples?|commands?|output|result)/i;
-
-// ── scoring ───────────────────────────────────────────────────────────────────
-
-function skillScore(raw, base) {
-  let score = 0;
-
-  // Fast path: frontmatter with name = definitely a skill
-  try {
-    const { data } = matter(raw);
-    if (data.name && typeof data.name === 'string') return 10;
-  } catch {}
-
-  const lines = raw.split('\n');
-  const nonEmpty = lines.filter(l => l.trim());
-  const headers = nonEmpty.filter(l => /^#{1,3}\s/.test(l));
-
-  // Minimum viable content
-  if (raw.length < 150)  return -99;
-  if (headers.length < 1) return -99;
-
-  // ── positive signals ──────────────────────────────────────────────────────
-
-  // Instructional section names (## Steps, ## Usage, etc.)
-  if (lines.some(l => INSTRUCTION_HEADERS.test(l))) score += 2;
-
-  // Headers with imperative verbs
-  if (headers.some(h => IMPERATIVE_HEADERS.test(h))) score += 2;
-
-  // Code block
-  if (raw.includes('```') || raw.includes('    ')) score += 1;
-
-  // Numbered list (step-by-step)
-  if (nonEmpty.some(l => /^\d+\.\s/.test(l))) score += 1;
-
-  // Bullet list
-  if (nonEmpty.some(l => /^[-*+]\s/.test(l))) score += 1;
-
-  // Multiple headers (structure)
-  if (headers.length >= 2) score += 1;
-  if (headers.length >= 4) score += 1;
-
-  // ── negative signals ──────────────────────────────────────────────────────
-
-  // First header looks like documentation
-  const firstHeader = headers[0]?.replace(/^#+\s*/, '') || '';
-  if (DOC_FIRST_HEADERS.test(firstHeader)) score -= 3;
-
-  // Content is mostly long prose paragraphs (narrative, not instructional)
-  const paragraphs = raw.split(/\n\n+/).filter(p => p.trim() && !p.trim().startsWith('#'));
-  const longProse = paragraphs.filter(p => p.split(' ').length > 60 && !/```/.test(p));
-  if (longProse.length > paragraphs.length * 0.6 && paragraphs.length > 3) score -= 2;
-
-  // Filename looks like a version, date, or index
-  if (SKIP_FILENAME_RE.test(base)) score -= 3;
-
-  // Very high word repetition (filler content)
-  const words = raw.toLowerCase().split(/\s+/).filter(w => w.length > 3);
-  if (words.length > 80) {
-    const unique = new Set(words);
-    if (unique.size / words.length < 0.22) score -= 2;
-  }
-
-  return score;
-}
-
-// ── public API ────────────────────────────────────────────────────────────────
-
 export function isSkillFile(filePath, raw) {
-  const parts = filePath.replace(/\\/g, '/').split('/');
-  const base = parts[parts.length - 1].replace(/\.md$/i, '').toLowerCase();
-
-  // Hard-reject by filename
-  if (SKIP_FILENAMES.has(base)) return false;
-  if (SKIP_FILENAME_RE.test(base)) return false;
-
-  // Hard-reject by parent directory
-  for (const part of parts.slice(0, -1)) {
-    if (SKIP_DIRS.has(part.toLowerCase())) return false;
-  }
+  const result = hardFilter(filePath, raw);
+  if (!result.pass) return false;
 
   try {
     if (!raw) raw = fs.readFileSync(filePath, 'utf8');
-
-    // Hard-reject: content starts with README header or badge lines
     const firstLines = raw.trimStart().slice(0, 300);
     if (/^#\s*readme\b/i.test(firstLines)) return false;
     if (/!\[.*\]\(https?:\/\/(img\.shields\.io|badge\.fury|travis-ci|github\.com\/[^)]+\/badge)/i.test(firstLines)) return false;
 
-    return skillScore(raw, base) >= 3;
+    return true;
   } catch {
-    return false;
+    return true;
   }
 }
 
+export async function filterWithClassifier(skills) {
+  const centroids = loadModel();
+  if (!centroids) return skills;
+
+  const { embedBatch } = await import('./embedder.js');
+  const texts = skills.map(s => s.content || '');
+  const embeddings = await embedBatch(texts);
+
+  return skills.filter((_, i) => {
+    const decision = classify(embeddings[i], centroids, skills[i].content, skills[i].path);
+    return decision.label === 'skill' || decision.label === 'unsure';
+  });
+}
+
 export function parseSkillFile(filePath, source, opts = {}) {
   const raw = opts.raw ?? fs.readFileSync(filePath, 'utf8');
 

package/registry/training/bad/accepts-HISTORY.md

@@ -0,0 +1,250 @@
+2.0.0 / 2024-08-31
+==================
+
+  * Drop node <18 support
+  * deps: mime-types@^3.0.0
+  * deps: negotiator@^1.0.0
+
+1.3.8 / 2022-02-02
+==================
+
+  * deps: mime-types@~2.1.34
+    - deps: mime-db@~1.51.0
+  * deps: negotiator@0.6.3
+
+1.3.7 / 2019-04-29
+==================
+
+  * deps: negotiator@0.6.2
+    - Fix sorting charset, encoding, and language with extra parameters
+
+1.3.6 / 2019-04-28
+==================
+
+  * deps: mime-types@~2.1.24
+    - deps: mime-db@~1.40.0
+
+1.3.5 / 2018-02-28
+==================
+
+  * deps: mime-types@~2.1.18
+    - deps: mime-db@~1.33.0
+
+1.3.4 / 2017-08-22
+==================
+
+  * deps: mime-types@~2.1.16
+    - deps: mime-db@~1.29.0
+
+1.3.3 / 2016-05-02
+==================
+
+  * deps: mime-types@~2.1.11
+    - deps: mime-db@~1.23.0
+  * deps: negotiator@0.6.1
+    - perf: improve `Accept` parsing speed
+    - perf: improve `Accept-Charset` parsing speed
+    - perf: improve `Accept-Encoding` parsing speed
+    - perf: improve `Accept-Language` parsing speed
+
+1.3.2 / 2016-03-08
+==================
+
+  * deps: mime-types@~2.1.10
+    - Fix extension of `application/dash+xml`
+    - Update primary extension for `audio/mp4`
+    - deps: mime-db@~1.22.0
+
+1.3.1 / 2016-01-19
+==================
+
+  * deps: mime-types@~2.1.9
+    - deps: mime-db@~1.21.0
+
+1.3.0 / 2015-09-29
+==================
+
+  * deps: mime-types@~2.1.7
+    - deps: mime-db@~1.19.0
+  * deps: negotiator@0.6.0
+    - Fix including type extensions in parameters in `Accept` parsing
+    - Fix parsing `Accept` parameters with quoted equals
+    - Fix parsing `Accept` parameters with quoted semicolons
+    - Lazy-load modules from main entry point
+    - perf: delay type concatenation until needed
+    - perf: enable strict mode
+    - perf: hoist regular expressions
+    - perf: remove closures getting spec properties
+    - perf: remove a closure from media type parsing
+    - perf: remove property delete from media type parsing
+
+1.2.13 / 2015-09-06
+===================
+
+  * deps: mime-types@~2.1.6
+    - deps: mime-db@~1.18.0
+
+1.2.12 / 2015-07-30
+===================
+
+  * deps: mime-types@~2.1.4
+    - deps: mime-db@~1.16.0
+
+1.2.11 / 2015-07-16
+===================
+
+  * deps: mime-types@~2.1.3
+    - deps: mime-db@~1.15.0
+
+1.2.10 / 2015-07-01
+===================
+
+  * deps: mime-types@~2.1.2
+    - deps: mime-db@~1.14.0
+
+1.2.9 / 2015-06-08
+==================
+
+  * deps: mime-types@~2.1.1
+    - perf: fix deopt during mapping
+
+1.2.8 / 2015-06-07
+==================
+
+  * deps: mime-types@~2.1.0
+    - deps: mime-db@~1.13.0
+  * perf: avoid argument reassignment & argument slice
+  * perf: avoid negotiator recursive construction
+  * perf: enable strict mode
+  * perf: remove unnecessary bitwise operator
+
+1.2.7 / 2015-05-10
+==================
+
+  * deps: negotiator@0.5.3
+    - Fix media type parameter matching to be case-insensitive
+
+1.2.6 / 2015-05-07
+==================
+
+  * deps: mime-types@~2.0.11
+    - deps: mime-db@~1.9.1
+  * deps: negotiator@0.5.2
+    - Fix comparing media types with quoted values
+    - Fix splitting media types with quoted commas
+
+1.2.5 / 2015-03-13
+==================
+
+  * deps: mime-types@~2.0.10
+    - deps: mime-db@~1.8.0
+
+1.2.4 / 2015-02-14
+==================
+
+  * Support Node.js 0.6
+  * deps: mime-types@~2.0.9
+    - deps: mime-db@~1.7.0
+  * deps: negotiator@0.5.1
+    - Fix preference sorting to be stable for long acceptable lists
+
+1.2.3 / 2015-01-31
+==================
+
+  * deps: mime-types@~2.0.8
+    - deps: mime-db@~1.6.0
+
+1.2.2 / 2014-12-30
+==================
+
+  * deps: mime-types@~2.0.7
+    - deps: mime-db@~1.5.0
+
+1.2.1 / 2014-12-30
+==================
+
+  * deps: mime-types@~2.0.5
+    - deps: mime-db@~1.3.1
+
+1.2.0 / 2014-12-19
+==================
+
+  * deps: negotiator@0.5.0
+    - Fix list return order when large accepted list
+    - Fix missing identity encoding when q=0 exists
+    - Remove dynamic building of Negotiator class
+
+1.1.4 / 2014-12-10
+==================
+
+  * deps: mime-types@~2.0.4
+    - deps: mime-db@~1.3.0
+
+1.1.3 / 2014-11-09
+==================
+
+  * deps: mime-types@~2.0.3
+    - deps: mime-db@~1.2.0
+
+1.1.2 / 2014-10-14
+==================
+
+  * deps: negotiator@0.4.9
+    - Fix error when media type has invalid parameter
+
+1.1.1 / 2014-09-28
+==================
+
+  * deps: mime-types@~2.0.2
+    - deps: mime-db@~1.1.0
+  * deps: negotiator@0.4.8
+    - Fix all negotiations to be case-insensitive
+    - Stable sort preferences of same quality according to client order
+
+1.1.0 / 2014-09-02
+==================
+
+  * update `mime-types`
+
+1.0.7 / 2014-07-04
+==================
+
+  * Fix wrong type returned from `type` when match after unknown extension
+
+1.0.6 / 2014-06-24
+==================
+
+  * deps: negotiator@0.4.7
+
+1.0.5 / 2014-06-20
+==================
+
+ * fix crash when unknown extension given
+
+1.0.4 / 2014-06-19
+==================
+
+  * use `mime-types`
+
+1.0.3 / 2014-06-11
+==================
+
+  * deps: negotiator@0.4.6
+    - Order by specificity when quality is the same
+
+1.0.2 / 2014-05-29
+==================
+
+  * Fix interpretation when header not in request
+  * deps: pin negotiator@0.4.5
+
+1.0.1 / 2014-01-18
+==================
+
+  * Identity encoding isn't always acceptable
+  * deps: negotiator@~0.4.0
+
+1.0.0 / 2013-12-27
+==================
+
+  * Genesis