promptgraph-mcp 2.4.7 → 2.5.0

package/github-import.js

@@ -4,15 +4,60 @@ import fs from 'fs';
 import https from 'https';
 import { globSync } from 'glob';
 import { indexAll, indexSource } from './indexer.js';
-import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './config.js';
+import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR, MAX_DOWNLOAD_SIZE, MAX_FILE_COUNT, MAX_REPO_SIZE, RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_MS } from './config.js';
 import { validateSkill } from './validator.js';
 import { isSkillFile } from './parser.js';
+import { RateLimiter } from './src/utils/rate-limiter.js';
+
+const githubRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS, windowMs: RATE_LIMIT_WINDOW_MS })
+const downloadRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS * 2, windowMs: RATE_LIMIT_WINDOW_MS })
 
 const SKILL_DIRS = ['skills', 'commands', 'prompts', 'agents', 'skills-store', 'slash-commands', 'custom-commands', 'templates'];
 
 // ── helpers ───────────────────────────────────────────────────────────────────
 
-function httpsGet(url) {
+const repoStats = new Map()
+
+function getRepoStats(ownerRepo) {
+  if (!repoStats.has(ownerRepo)) {
+    repoStats.set(ownerRepo, { totalBytes: 0, fileCount: 0 })
+  }
+  return repoStats.get(ownerRepo)
+}
+
+function streamDownload(url, maxSize = MAX_DOWNLOAD_SIZE) {
+  return new Promise((res, rej) => {
+    const token = process.env.GITHUB_TOKEN;
+    const headers = { 'User-Agent': 'promptgraph-mcp' };
+    if (token && url.startsWith('https://raw.')) headers['Authorization'] = `Bearer ${token}`;
+    const req = https.get(url, { headers }, r => {
+      if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location)
+        return streamDownload(r.headers.location, maxSize).then(res, rej);
+      if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
+      const cl = parseInt(r.headers['content-length'], 10);
+      if (!isNaN(cl) && cl > maxSize) {
+        r.resume();
+        return rej(new Error(`Content-Length ${cl} exceeds max ${maxSize}`));
+      }
+      let d = ''
+      let total = 0
+      r.setEncoding('utf8')
+      r.on('data', c => {
+        total += Buffer.byteLength(c, 'utf8')
+        if (total > maxSize) {
+          r.destroy()
+          return rej(new Error(`Download exceeded ${maxSize} bytes`))
+        }
+        d += c
+      })
+      r.on('end', () => res(d))
+    })
+    req.on('error', rej)
+  })
+}
+
+async function httpsGet(url) {
+  await githubRateLimiter.acquire()
   const token = process.env.GITHUB_TOKEN;
   const headers = { 'User-Agent': 'promptgraph-mcp' };
   if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
@@ -39,12 +84,24 @@ function repoExists(ownerRepo) {
 }
 
 // Download one .md file, run validateSkill on it, return errors/warnings.
-async function validateMdFile(file, tmpDir) {
+async function validateMdFile(file, tmpDir, ownerRepo) {
   const errors = [];
   const warnings = [];
+  const stats = getRepoStats(ownerRepo);
   try {
-    const content = await httpsGet(file.download_url);
+    if (stats.fileCount >= MAX_FILE_COUNT) {
+      errors.push(`${file.name}: skipped — repo file count limit (${MAX_FILE_COUNT}) reached`);
+      return { errors, warnings };
+    }
+    await downloadRateLimiter.acquire()
+    const content = await streamDownload(file.download_url);
+    stats.totalBytes += Buffer.byteLength(content, 'utf8');
+    stats.fileCount++;
+    if (stats.totalBytes > MAX_REPO_SIZE) {
+      return { errors: [...errors, `${file.name}: repo size limit (${MAX_REPO_SIZE}) exceeded`], warnings };
+    }
     const tmpPath = path.join(tmpDir, file.name);
+    fs.mkdirSync(path.dirname(tmpPath), { recursive: true });
     fs.writeFileSync(tmpPath, content);
     const result = validateSkill(tmpPath);
     if (!result.ok) {
@@ -70,17 +127,28 @@ export async function validateRepoSkills(ownerRepo) {
 
   let mdFiles;
   if (detected) {
-    // Has a skills subdirectory — list contents
+    // Has a skills subdirectory — use git tree API for recursive listing
     const subdir = detected.subdir;
-    let entries;
     try {
-      const json = await httpsGet(`https://api.github.com/repos/${ownerRepo}/contents/${subdir}`);
-      entries = JSON.parse(json);
+      const treeJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}/git/trees/HEAD?recursive=1`);
+      const tree = JSON.parse(treeJson);
+      const prefix = subdir + '/';
+      const mdTreeEntries = (tree.tree || []).filter(f =>
+        f.type === 'blob' && f.path.startsWith(prefix) && f.path.endsWith('.md')
+      );
+      let branch = 'main';
+      try {
+        const repoJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}`);
+        branch = JSON.parse(repoJson).default_branch || 'main';
+      } catch {}
+      mdFiles = mdTreeEntries.map(f => ({
+        name: f.path.replace(prefix, ''),
+        download_url: `https://raw.githubusercontent.com/${ownerRepo}/${branch}/${f.path}`
+      }));
     } catch (e) {
       try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
       return { ok: false, errors: [`Failed to list ${subdir}/ contents: ${e.message}`], warnings: [] };
     }
-    mdFiles = entries.filter(e => e.type === 'file' && e.name.endsWith('.md'));
   } else {
     // No skills subdir — fall back to root-level .md files
     try {
@@ -107,7 +175,7 @@ export async function validateRepoSkills(ownerRepo) {
   let warnings = [];
 
   for (const file of mdToValidate) {
-    const r = await validateMdFile(file, tmpDir);
+    const r = await validateMdFile(file, tmpDir, ownerRepo);
     errors.push(...r.errors);
     warnings.push(...r.warnings);
   }
@@ -131,6 +199,20 @@ async function detectSkillsDirFromAPI(ownerRepo) {
       if (dirMap.has(d)) return { subdir: dirMap.get(d), label: d };
     }
 
+    // 1.5 Nested skills dirs (e.g. .claude/skills, .claude-plugin/commands)
+    for (const prefix of ['.claude', '.claude-plugin']) {
+      if (dirMap.has(prefix)) {
+        for (const d of SKILL_DIRS) {
+          const nested = `${prefix}/${d}`;
+          try {
+            const sub = await httpsGet(`https://api.github.com/repos/${ownerRepo}/contents/${nested}`);
+            const subEntries = JSON.parse(sub);
+            if (subEntries.length > 0) return { subdir: nested, label: nested };
+          } catch {}
+        }
+      }
+    }
+
     // 2. Any subdir with 2+ .md files — pick the one with most .md files
     const subdirCandidates = entries.filter(e => e.type === 'dir' && !SKIP_DIRS_API.has(e.name.toLowerCase()));
     let best = null, bestCount = 0;
@@ -152,6 +234,8 @@ const SKIP_DIRS_API = new Set([
   '.github', 'docs', 'doc', 'documentation', 'examples', 'example',
   'tests', 'test', 'assets', 'images', 'img', 'media', 'static',
   'node_modules', 'vendor', 'dist', 'build', '.git',
+  'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
+  'cheatsheets', 'resources',
 ]);
 
 function git(args, cwd, stdio = 'inherit') {
@@ -177,16 +261,23 @@ function sparseClone(url, dest, subdir) {
   // Try HEAD, then main, then master
   for (const branch of ['HEAD', 'main', 'master']) {
     const r = git(['checkout', branch === 'HEAD' ? 'FETCH_HEAD' : branch], dest, 'pipe');
-    if (r.status === 0) return true;
+    if (r.status === 0) return finalizeCheckout(dest, true);
   }
   return false;
 }
 
+// Shared skip patterns — module scope so both cleanup functions can access them
+const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
+const SKIP_DIRS_LOCAL = new Set([
+  '.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots',
+  'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor',
+  'dist', 'build', 'tests', 'test',
+  'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
+  'cheatsheets', 'resources',
+]);
+
 // After full-clone root: remove files that are not skills and dirs we don't need
 function cleanupRepoRoot(repoRoot) {
-  const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
-  const SKIP_DIRS_LOCAL = new Set(['.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots', 'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor', 'dist', 'build', 'tests', 'test']);
-
   let removed = 0;
   const entries = fs.readdirSync(repoRoot, { withFileTypes: true });
   for (const entry of entries) {
@@ -223,9 +314,14 @@ function cleanupRepoDir(dirPath, SKIP_RE) {
   for (const entry of entries) {
     const fullPath = path.join(dirPath, entry.name);
     if (entry.isDirectory()) {
-      removed += cleanupRepoDir(fullPath, SKIP_RE);
-      // Remove empty dirs after cleanup
-      try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
+      // Remove entire skip dirs (e.g. references/) nested inside skills dirs
+      if (SKIP_DIRS_LOCAL.has(entry.name.toLowerCase())) {
+        fs.rmSync(fullPath, { recursive: true, force: true });
+        removed++;
+      } else {
+        removed += cleanupRepoDir(fullPath, SKIP_RE);
+        try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
+      }
     } else if (entry.isFile() && entry.name.endsWith('.md')) {
       const base = entry.name.replace(/\.md$/i, '').toLowerCase();
       if (SKIP_RE.test(base)) {
@@ -252,28 +348,42 @@ function removeEmptyDirs(dirPath) {
   }
 }
 
-// Update sparse repo — fetch + reset
+// After fetch/checkout: force materialization of all sparse-matched files.
+// partial clone (blob:none) + checkout often skips blob download on Windows.
+function forceMaterialize(dest) {
+  git(['checkout', 'HEAD', '--', '.'], dest, 'pipe');
+}
+
+// Update sparse repo — fetch + checkout
 function sparseUpdate(dest, subdir) {
   const fetch = git(['fetch', '--depth=1', 'origin'], dest);
   if (fetch.status !== 0) return false;
 
-  // Ensure sparse-checkout still set correctly
   git(['sparse-checkout', 'set', '--no-cone', `${subdir}/*.md`, `${subdir}/**/*.md`], dest, 'pipe');
 
-  for (const ref of ['origin/HEAD', 'origin/main', 'origin/master']) {
-    const r = git(['reset', '--hard', ref], dest, 'pipe');
-    if (r.status === 0) return true;
+  for (const ref of ['origin/main', 'origin/master']) {
+    const r = git(['checkout', ref], dest, 'pipe');
+    if (r.status !== 0) continue;
+    forceMaterialize(dest);
+    return true;
   }
   return false;
 }
 
+// After checkout, force materialization of sparse-matched files.
+function finalizeCheckout(dest, success) {
+  if (success) forceMaterialize(dest);
+  return success;
+}
+
 // Fallback: clone root but only checkout .md files
 function fullClone(url, dest) {
   if (fs.existsSync(dest)) {
     const fetch = git(['fetch', '--depth=1', '--filter=blob:none', 'origin'], dest);
     if (fetch.status !== 0) return false;
     for (const ref of ['origin/HEAD', 'origin/main', 'origin/master']) {
-      if (git(['reset', '--hard', ref], dest, 'pipe').status === 0) return true;
+      const ok = git(['reset', '--hard', ref], dest, 'pipe').status === 0;
+      if (ok) return finalizeCheckout(dest, true);
     }
     return false;
   }
@@ -287,13 +397,14 @@ function fullClone(url, dest) {
   if (fetch.status !== 0) return false;
   for (const branch of ['FETCH_HEAD', 'main', 'master']) {
     const r = git(['checkout', branch], dest, 'pipe');
-    if (r.status === 0) return true;
+    if (r.status === 0) return finalizeCheckout(dest, true);
   }
   return false;
 }
 
-// After clone: detect actual skills dir on disk
+// After clone: detect actual skills dir on disk (flat + nested)
 function detectSkillsDirLocal(repoRoot) {
+  // 1. Flat dirs matching SKILL_DIRS (e.g. skills/, commands/)
   for (const dir of SKILL_DIRS) {
     const candidate = path.join(repoRoot, dir);
     if (fs.existsSync(candidate)) {
@@ -301,6 +412,16 @@ function detectSkillsDirLocal(repoRoot) {
       if (files.length >= 1) return { dir: candidate, label: dir, sparse: true };
     }
   }
+  // 2. Nested dirs: .claude/skills, .claude-plugin/commands, etc.
+  for (const prefix of ['.claude', '.claude-plugin']) {
+    for (const dir of SKILL_DIRS) {
+      const candidate = path.join(repoRoot, prefix, dir);
+      if (fs.existsSync(candidate)) {
+        const files = globSync(`${candidate}/**/*.md`);
+        if (files.length >= 1) return { dir: candidate, label: `${prefix}/${dir}`, sparse: true };
+      }
+    }
+  }
   return { dir: repoRoot, label: '(root)', sparse: false };
 }
 
@@ -356,7 +477,17 @@ export async function importFromGitHub(repoUrl) {
     const sparseList = isSparse
       ? spawnSync('git', ['sparse-checkout', 'list'], { cwd: dest, encoding: 'utf8' }).stdout.trim()
       : '';
-    skillsSubdir = sparseList.split('\n').find(l => SKILL_DIRS.includes(l.trim())) || null;
+    skillsSubdir = (() => {
+      const lines = sparseList.split('\n').map(l => l.trim()).filter(Boolean);
+      if (lines.length === 0) return null;
+      const exact = lines.find(l => SKILL_DIRS.includes(l));
+      if (exact) return exact;
+      for (const line of lines) {
+        const m = line.match(/^(.+)\/(?:\*\*\/)?\*\.md$/);
+        if (m && !m[1].includes('*')) return m[1];
+      }
+      return null;
+    })();
 
     cloneOk = skillsSubdir
       ? sparseUpdate(dest, skillsSubdir)
@@ -381,9 +512,18 @@ export async function importFromGitHub(repoUrl) {
     removeEmptyDirs(dest);
   }
 
-  const { dir: skillsDir, label } = detectSkillsDirLocal(dest);
+  const { dir: localDir, label: localLabel } = detectSkillsDirLocal(dest);
+  // Prefer the known skillsSubdir (from API detection or sparse patterns) as the
+  // canonical skills directory — it's more accurate than local heuristics for
+  // repos with non-standard dir names (e.g. "specialized", "cli", or nested paths)
+  const skillsDir = skillsSubdir ? path.join(dest, skillsSubdir) : localDir;
+  const label = skillsSubdir || localLabel;
   const mdFiles = globSync(`${skillsDir}/**/*.md`);
 
+  if (mdFiles.length > MAX_FILE_COUNT) {
+    console.warn(`Warning: ${mdFiles.length} .md files exceeds limit of ${MAX_FILE_COUNT} — truncating`);
+  }
+
   if (skillsSubdir) {
     console.log(`Sparse-checkout: ${label}/ only (${mdFiles.length} .md files, no other repo files)`);
   } else {

package/indexer.js

@@ -3,7 +3,8 @@ import { createHash } from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import { parseSkillFile, isSkillFile, filterWithClassifier } from './parser.js';
-import { embedBatch, cosineSimilarity, BATCH_SIZE } from './embedder.js';
+import { embedBatch, cosineSimilarity } from './embedder.js';
+import { BATCH_SIZE } from './config.js';
 import { getDb, skillId, vecToBlob } from './db.js';
 import { loadConfig } from './config.js';
 import { chunkText } from './chunker.js';
@@ -11,16 +12,29 @@ import { buildAnnIndex } from './ann.js';
 import { progress, progressDone, success, info, spinner } from './cli.js';
 import chalk from 'chalk';
 
+const MAX_FILE_SIZE = 5 * 1024 * 1024;   // 5 MB per file
+const MAX_FILE_COUNT = 100000;            // 100k files per reindex
+
+function sanitizePath(filePath) {
+  return path.resolve(filePath);
+}
+
 async function indexBatch(db, skills, { fast = false } = {}) {
   const upsertSkill = db.prepare(`
-    INSERT INTO skills (id, name, description, path, source, content, hash)
-    VALUES (@id, @name, @description, @path, @source, @content, @hash)
+    INSERT INTO skills (id, name, description, path, source, content, hash, version, author, license, updated_at, downloads, verified)
+    VALUES (@id, @name, @description, @path, @source, @content, @hash, @version, @author, @license, @updated_at, @downloads, @verified)
     ON CONFLICT(id) DO UPDATE SET
       name = excluded.name,
       description = excluded.description,
       path = excluded.path,
       content = excluded.content,
-      hash = excluded.hash
+      hash = excluded.hash,
+      version = excluded.version,
+      author = excluded.author,
+      license = excluded.license,
+      updated_at = excluded.updated_at,
+      downloads = excluded.downloads,
+      verified = excluded.verified
   `);
   const deleteChunks = db.prepare('DELETE FROM chunks WHERE skill_id = ?');
   const deleteEdges = db.prepare('DELETE FROM edges WHERE from_skill = ?');
@@ -51,7 +65,7 @@ async function indexBatch(db, skills, { fast = false } = {}) {
   db.transaction(() => {
     for (const skill of skills) {
       const id = skillId(skill.source, skill.name);
-      upsertSkill.run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content, hash: skill.hash || null });
+      upsertSkill.run({ id, name: skill.name, description: skill.description, path: skill.path, source: skill.source, content: skill.content, hash: skill.hash || null, version: skill.version || null, author: skill.author || null, license: skill.license || null, updated_at: skill.updated_at || null, downloads: skill.downloads ?? 0, verified: skill.verified ? 1 : 0 });
       upsertFts.run(id, skill.name, skill.description || '', skill.content || '');
       if (!fast) {
         deleteChunks.run(id);
@@ -93,19 +107,30 @@ export async function indexAll({ fast = false } = {}) {
     normDir: path.resolve(s.dir),
   })).sort((a, b) => b.normDir.length - a.normDir.length); // longest first
 
+  console.error('DEBUG normalizedSources count:', normalizedSources.length);
+  console.error('DEBUG first dir:', normalizedSources[0]?.dir);
+
   const seenFiles = new Set();
   const allFiles = [];
   for (const { dir, source } of normalizedSources) {
     const files = globSync(`${dir}/**/*.md`);
+    console.error('DEBUG source:', source, 'dir:', dir, 'count:', files.length);
     for (const f of files) {
-      const norm = path.resolve(f);
+      const norm = sanitizePath(f);
       if (!seenFiles.has(norm)) {
         seenFiles.add(norm);
-        allFiles.push({ file: f, source });
+        allFiles.push({ file: norm, source });
+      } else {
+        console.error('DEBUG duplicate:', norm);
       }
     }
+    if (allFiles.length > MAX_FILE_COUNT) {
+      info(chalk.yellow(`Reached max file count (${MAX_FILE_COUNT}) — truncating`));
+      break;
+    }
   }
   const total = allFiles.length;
+  console.error('DEBUG total files found:', total);
   info(`Found ${chalk.white.bold(total)} files`);
 
   // reconcile: remove skills whose files no longer exist OR whose name changed
@@ -150,9 +175,13 @@ export async function indexAll({ fast = false } = {}) {
 
   for (const { file, source } of allFiles) {
     try {
-      // 1. Read file once
+      // 1. Read file once (with size gate)
       let raw;
-      try { raw = fs.readFileSync(file, 'utf8'); } catch { skipped++; count++; continue; }
+      try {
+        const stat = fs.statSync(file);
+        if (stat.size > MAX_FILE_SIZE) { skipped++; count++; continue; }
+        raw = fs.readFileSync(file, 'utf8');
+      } catch { skipped++; count++; continue; }
 
       // 2. Hash first — cheapest check
       const hash = createHash('md5').update(raw).digest('hex');
@@ -182,6 +211,7 @@ export async function indexAll({ fast = false } = {}) {
         batch = [];
         const eta = count > 0 ? Math.round((total - count) * (Date.now() - start) / count / 1000) : '?';
         progress(count, total, { skipped, eta, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) {
       errors++;
@@ -222,10 +252,13 @@ export async function indexAll({ fast = false } = {}) {
 }
 
 export async function indexFile(filePath, source) {
+  const safe = sanitizePath(filePath);
+  const stat = fs.statSync(safe);
+  if (stat.size > MAX_FILE_SIZE) throw new Error(`File too large: ${filePath}`);
   const db = getDb();
-  const raw = fs.readFileSync(filePath, 'utf8');
+  const raw = fs.readFileSync(safe, 'utf8');
   const hash = createHash('md5').update(raw).digest('hex');
-  const skill = parseSkillFile(filePath, source, { raw });
+  const skill = parseSkillFile(safe, source, { raw });
   await indexBatch(db, [{ ...skill, hash }]);
 }
 
@@ -256,7 +289,10 @@ export async function indexSource(dir, sourceName) {
 
   for (const file of files) {
     try {
-      const raw = fs.readFileSync(file, 'utf8');
+      const norm = sanitizePath(file);
+      const stat = fs.statSync(norm);
+      if (stat.size > MAX_FILE_SIZE) { skipped++; count++; continue; }
+      const raw = fs.readFileSync(norm, 'utf8');
       const hash = createHash('md5').update(raw).digest('hex');
       const dbRow = dbByPath.get(file);
       if (dbRow?.hash === hash) { skipped++; count++; continue; }
@@ -267,6 +303,7 @@ export async function indexSource(dir, sourceName) {
         await indexBatch(db, batch);
         count += batch.length; batch = [];
         progress(count, total, { skipped, errors });
+        await new Promise(r => setImmediate ? setImmediate(r) : setTimeout(r, 0));
       }
     } catch (e) { errors++; count++; }
   }