promptgraph-mcp 2.4.4 → 2.4.6

package/bundle-counts.js

@@ -16,8 +16,11 @@ const SKIP_ROOT_DIRS = new Set(['.github', 'docs', 'doc', 'assets', 'images', 'i
 const SKIP_NAMES = /^(readme|changelog|license|contributing|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
 
 function httpsGet(url) {
+  const token = process.env.GITHUB_TOKEN;
+  const headers = { 'User-Agent': 'promptgraph-mcp' };
+  if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
   return new Promise((res, rej) => {
-    const req = https.get(url, { headers: { 'User-Agent': 'promptgraph-mcp' } }, r => {
+    const req = https.get(url, { headers }, r => {
       if (r.statusCode === 403 || r.statusCode === 429) return rej(new Error(`Rate limited`));
       if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
       let d = ''; r.setEncoding('utf8'); r.on('data', c => d += c); r.on('end', () => res(d));

package/github-import.js

@@ -12,8 +12,11 @@ const SKILL_DIRS = ['skills', 'commands', 'prompts', 'agents', 'skills-store', '
 // ── helpers ───────────────────────────────────────────────────────────────────
 
 function httpsGet(url) {
+  const token = process.env.GITHUB_TOKEN;
+  const headers = { 'User-Agent': 'promptgraph-mcp' };
+  if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
   return new Promise((res, rej) => {
-    const req = https.get(url, { headers: { 'User-Agent': 'promptgraph-mcp' } }, r => {
+    const req = https.get(url, { headers }, r => {
       if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location)
         return httpsGet(r.headers.location).then(res, rej);
       if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
@@ -192,6 +195,9 @@ function cleanupRepoRoot(repoRoot) {
       if (SKIP_DIRS_LOCAL.has(entry.name.toLowerCase())) {
         fs.rmSync(fullPath, { recursive: true, force: true });
         removed++;
+      } else {
+        // Recurse into subdirectory to remove doc files
+        removed += cleanupRepoDir(fullPath, SKIP_RE);
       }
     } else if (entry.isFile() && entry.name.endsWith('.md')) {
       const base = entry.name.replace(/\.md$/i, '').toLowerCase();
@@ -200,13 +206,49 @@ function cleanupRepoRoot(repoRoot) {
         removed++;
       }
     } else if (entry.isFile() && !entry.name.endsWith('.md')) {
-      // Remove non-md files (gitignore, LICENSE, Makefile, etc.) — keep only .md
       if (entry.name !== '.gitignore') {
         try { fs.unlinkSync(fullPath); removed++; } catch {}
       }
     }
   }
-  if (removed > 0) console.log(`Cleaned up ${removed} non-skill files/dirs from root`);
+  if (removed > 0) console.log(`Cleaned up ${removed} non-skill files/dirs`);
+}
+
+// Recursively remove doc .md files from subdirectories
+function cleanupRepoDir(dirPath, SKIP_RE) {
+  let removed = 0;
+  let entries;
+  try { entries = fs.readdirSync(dirPath, { withFileTypes: true }); } catch { return 0; }
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name);
+    if (entry.isDirectory()) {
+      removed += cleanupRepoDir(fullPath, SKIP_RE);
+      // Remove empty dirs after cleanup
+      try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
+    } else if (entry.isFile() && entry.name.endsWith('.md')) {
+      const base = entry.name.replace(/\.md$/i, '').toLowerCase();
+      if (SKIP_RE.test(base)) {
+        fs.unlinkSync(fullPath);
+        removed++;
+      }
+    } else if (entry.isFile() && !entry.name.endsWith('.md')) {
+      try { fs.unlinkSync(fullPath); removed++; } catch {}
+    }
+  }
+  return removed;
+}
+
+// Recursively remove empty directories
+function removeEmptyDirs(dirPath) {
+  let entries;
+  try { entries = fs.readdirSync(dirPath, { withFileTypes: true }); } catch { return; }
+  for (const entry of entries) {
+    if (entry.name === '.git') continue;
+    if (!entry.isDirectory()) continue;
+    const fullPath = path.join(dirPath, entry.name);
+    removeEmptyDirs(fullPath);
+    try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
+  }
 }
 
 // Update sparse repo — fetch + reset
@@ -321,6 +363,24 @@ export async function importFromGitHub(repoUrl) {
     if (!cloneOk) throw new Error(`Update failed for ${repoName}`);
   }
 
+  // Remove doc files anywhere in the cloned tree
+  cleanupRepoRoot(dest);
+
+  // Validate every .md file — delete those that fail
+  const allMd = globSync(`${dest}/**/*.md`);
+  let removedInvalid = 0;
+  for (const fp of allMd) {
+    const v = validateSkill(fp);
+    if (!v.ok) {
+      try { fs.unlinkSync(fp); removedInvalid++; } catch {}
+    }
+  }
+  if (removedInvalid > 0) {
+    console.log(`Removed ${removedInvalid} invalid .md files (failed validation)`);
+    // Clean up empty dirs left behind
+    removeEmptyDirs(dest);
+  }
+
   const { dir: skillsDir, label } = detectSkillsDirLocal(dest);
   const mdFiles = globSync(`${skillsDir}/**/*.md`);
 

package/marketplace.js

@@ -11,6 +11,7 @@ import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './con
 import { importFromGitHub, validateRepoSkills } from './github-import.js';
 
 const REGISTRY_URL = 'https://raw.githubusercontent.com/NeiP4n/promptgraph-registry/main/registry.json';
+const SKILL_COUNT_CACHE = path.join(PROMPTGRAPH_DIR, 'skill-counts.json');
 const SKILLS_DIR = path.join(SKILLS_STORE_DIR, 'marketplace');
 
 // Atomically write content to dest via tmp — cleans up on failure
@@ -81,6 +82,29 @@ async function rawFetch(url) {
 
 // Disk cache for the registry (network to GitHub raw can be slow on some networks).
 const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+const SKILL_COUNT_TTL = 24 * 60 * 60 * 1000; // 24 hours for skill counts
+
+// Return GitHub API auth headers if GITHUB_TOKEN is set
+function githubAuthHeaders() {
+  const token = process.env.GITHUB_TOKEN;
+  if (!token) return { 'User-Agent': 'promptgraph-mcp' };
+  return { 'User-Agent': 'promptgraph-mcp', 'Authorization': `Bearer ${token}` };
+}
+
+// Read skill count cache, returns {} on miss
+function readSkillCountCache() {
+  try {
+    if (fs.existsSync(SKILL_COUNT_CACHE)) return JSON.parse(fs.readFileSync(SKILL_COUNT_CACHE, 'utf8'));
+  } catch {}
+  return {};
+}
+
+function writeSkillCountCache(data) {
+  try {
+    fs.mkdirSync(path.dirname(SKILL_COUNT_CACHE), { recursive: true });
+    fs.writeFileSync(SKILL_COUNT_CACHE, JSON.stringify(data));
+  } catch {}
+}
 
 async function fetchText(url) {
   const cacheFile = path.join(PROMPTGRAPH_DIR, 'registry-cache.json');
@@ -187,7 +211,7 @@ function isSkillFile(path) {
 async function countRepoSkills(repoUrl) {
   try {
     const apiUrl = `https://api.github.com/repos/${repoUrl}/git/trees/HEAD?recursive=1`;
-    const res = await fetch(apiUrl, { headers: { 'User-Agent': 'promptgraph-mcp' } });
+    const res = await fetch(apiUrl, { headers: githubAuthHeaders() });
     if (!res.ok) return null;
     const data = await res.json();
     return (data.tree || []).filter(f => f.type === 'blob' && isSkillFile(f.path)).length;
@@ -199,12 +223,33 @@ export async function browseBundles(topK = 20) {
     const text = await fetchText(REGISTRY_URL);
     const registry = JSON.parse(text);
     const bundles = registry.bundles || [];
+    const cache = readSkillCountCache();
+    const now = Date.now();
+    let changed = false;
+
     await Promise.all(bundles.map(async b => {
-      // Re-count all repo bundles with the new .md-only filter
-      if (b.repo_url) b.skillCount = await countRepoSkills(b.repo_url) ?? 0;
+      if (!b.repo_url) return;
+      const cached = cache[b.repo_url];
+      // Use cached count if fresh, else fetch from API
+      if (cached && (now - cached.ts) < SKILL_COUNT_TTL) {
+        b.skillCount = cached.count;
+      } else {
+        const count = await countRepoSkills(b.repo_url);
+        if (count !== null) {
+          b.skillCount = count;
+          cache[b.repo_url] = { count, ts: now };
+          changed = true;
+        } else {
+          // API failed — use stale cache if exists, else keep registry value as fallback
+          b.skillCount = cached?.count ?? b.skillCount ?? 0;
+        }
+      }
     }));
+
+    if (changed) writeSkillCountCache(cache);
+
     return bundles
-      .filter(b => !b.repo_url || b.skillCount > 0) // hide bundles with 0 .md files
+      .filter(b => !b.repo_url || b.skillCount > 0)
       .map(b => ({ ...b, code: b.code || codeFor(b.id) }))
       .sort((a, b) => (b.stars || 0) - (a.stars || 0))
       .slice(0, topK);
@@ -322,16 +367,17 @@ export async function publishBundle(bundleDef) {
     return { error: 'Bundle validation failed', issues: validation.errors, warnings: validation.warnings };
   }
 
-  // Validate repo skills content if repo_url is set
+  // Best-effort repo validation (client-side). GitHub Actions is the source of truth.
+  let repoWarnings = [];
   if (def.repo_url) {
-    const ownerRepo = def.repo_url.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '');
-    const repoValidation = await validateRepoSkills(ownerRepo);
-    if (!repoValidation.ok) {
-      return {
-        error: 'Repo skills validation failed',
-        issues: repoValidation.errors,
-        warnings: repoValidation.warnings,
-      };
+    try {
+      const ownerRepo = def.repo_url.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '');
+      const repoValidation = await validateRepoSkills(ownerRepo);
+      if (!repoValidation.ok) {
+        repoWarnings = repoValidation.errors;
+      }
+    } catch (e) {
+      repoWarnings = [`Repo validation warning: ${e.message} (will be checked by CI)`];
     }
   }
 
@@ -345,6 +391,7 @@ export async function publishBundle(bundleDef) {
 
   if (gh.no_gh) {
     const issueUrl = `${REGISTRY_ISSUES}?title=Bundle%3A+${encodeURIComponent(def.name)}&body=${encodeURIComponent('Bundle definition:\n\n```json\n' + bundleJson + '\n```')}`;
+    const actionNote = def.repo_url ? `\n\nNote: Your repo will be validated by CI (GitHub Actions) after submission.\nRun locally: node validate-repo-action.js ${def.repo_url}` : '';
     return {
       success: true,
       gh_not_installed: true,
@@ -352,6 +399,8 @@ export async function publishBundle(bundleDef) {
         '1. Install gh CLI: https://cli.github.com',
         `   OR open this pre-filled issue directly: ${issueUrl}`,
         '2. Paste the bundle JSON shown below into the issue body',
+        ...(repoWarnings.length ? ['', '⚠ Repo warnings (CI will re-check):', ...repoWarnings.map(w => '   - ' + w)] : []),
+        ...(def.repo_url ? ['', 'Your repo will be validated by CI when submitted.'] : []),
       ].join('\n'),
       bundle_json: bundleJson,
       submit_url: issueUrl,
@@ -359,7 +408,10 @@ export async function publishBundle(bundleDef) {
   }
   if (!gh.ok) return { error: gh.error };
   const issueUrl = `${REGISTRY_ISSUES}?title=Bundle%3A+${encodeURIComponent(def.name)}&body=Gist%3A+${encodeURIComponent(gh.url)}`;
-  return { success: true, gist_url: gh.url, submit_url: issueUrl, message: `Bundle published! Submit: ${issueUrl}` };
+  const msg = def.repo_url
+    ? `Bundle published! Submit: ${issueUrl}\n\nRepo will be validated by CI. Run: node validate-repo-action.js ${def.repo_url}`
+    : `Bundle published! Submit: ${issueUrl}`;
+  return { success: true, gist_url: gh.url, submit_url: issueUrl, message: msg };
 }
 
 export function getTopRated(topK = 10) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.4.4",
+  "version": "2.4.6",
   "main": "index.js",
   "type": "module",
   "bin": {

package/validate-repo-action.js

@@ -0,0 +1,139 @@
+import fs from 'fs';
+import path from 'path';
+import { spawnSync } from 'child_process';
+import { validateSkill } from './validator.js';
+
+// ── doc filename filter — same as cleanupRepoRoot in github-import.js ──────────
+const SKIP_RE = /^(readme|changelog|license|contributing|code\.?of\.?conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding|changelog)/i;
+
+const SKIP_DIRS_LOCAL = new Set([
+  '.github', 'docs', 'doc', 'documentation', 'assets', 'images', 'img',
+  'screenshots', 'media', 'static', 'scripts', 'ci_scripts',
+  'node_modules', 'vendor', 'dist', 'build', 'tests', 'test',
+  'examples', 'example', 'fixtures',
+]);
+
+function isDocFile(name) {
+  const base = path.basename(name, '.md').toLowerCase();
+  return SKIP_RE.test(base);
+}
+
+function isSkipDir(name) {
+  return SKIP_DIRS_LOCAL.has(name.toLowerCase());
+}
+
+// ── find .md files in subdirectories only (never root) ────────────────────────
+function findSubdirMdFiles(repoRoot) {
+  const files = [];
+  const entries = fs.readdirSync(repoRoot, { withFileTypes: true });
+
+  for (const entry of entries) {
+    if (entry.name === '.git') continue;
+    const fullPath = path.join(repoRoot, entry.name);
+
+    if (entry.isDirectory()) {
+      if (isSkipDir(entry.name)) continue;
+      // Walk subdirectory recursively
+      walkDir(fullPath, entry.name, files);
+    }
+    // Root files are SKIPPED — never count them
+  }
+  return files;
+}
+
+function walkDir(dirPath, relativePrefix, out) {
+  let entries;
+  try { entries = fs.readdirSync(dirPath, { withFileTypes: true }); } catch { return; }
+
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name);
+    const relativePath = relativePrefix + '/' + entry.name;
+
+    if (entry.isDirectory()) {
+      if (isSkipDir(entry.name)) continue;
+      walkDir(fullPath, relativePath, out);
+    } else if (entry.isFile() && entry.name.endsWith('.md')) {
+      if (isDocFile(entry.name)) continue;
+      out.push({ path: fullPath, relative: relativePath });
+    }
+  }
+}
+
+// ── main ──────────────────────────────────────────────────────────────────────
+async function main() {
+  const args = process.argv.slice(2);
+  const repoArg = args.find(a => a.startsWith('--repo='))?.split('=').slice(1).join('=') || args[0];
+
+  if (!repoArg) {
+    console.error(JSON.stringify({ ok: false, errors: ['Usage: node validate-repo-action.js <owner/repo>'] }));
+    process.exit(1);
+  }
+
+  const repo = repoArg.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '').replace(/\/$/, '');
+  const cloneUrl = `https://github.com/${repo}.git`;
+  const tmpDir = path.join(process.env.RUNNER_TEMP || process.env.TMPDIR || '/tmp', `validate-${repo.replace('/', '-')}`);
+
+  // Clean any previous run
+  if (fs.existsSync(tmpDir)) fs.rmSync(tmpDir, { recursive: true, force: true });
+
+  // Clone with depth 1 (no API calls, no rate limits)
+  const gitEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0' };
+  const clone = spawnSync('git', ['clone', '--depth=1', cloneUrl, tmpDir], {
+    stdio: 'pipe', env: gitEnv,
+  });
+
+  if (clone.status !== 0) {
+    const msg = (clone.stderr?.toString() || 'unknown error').trim();
+    console.error(JSON.stringify({ ok: false, errors: [`Failed to clone ${repo}: ${msg}`] }));
+    if (fs.existsSync(tmpDir)) fs.rmSync(tmpDir, { recursive: true, force: true });
+    process.exit(1);
+  }
+
+  // Find .md files in subdirectories only
+  const candidates = findSubdirMdFiles(tmpDir);
+
+  if (candidates.length === 0) {
+    console.error(JSON.stringify({
+      ok: false,
+      errors: ['No valid .md skill files found in subdirectories'],
+      detail: 'Root-level .md files are ignored. Skills must be in a subdirectory (skills/, prompts/, etc.).',
+    }));
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+    process.exit(1);
+  }
+
+  // Validate each .md file via validateSkill()
+  const results = [];
+  let totalErrors = 0;
+
+  for (const file of candidates) {
+    const result = validateSkill(file.path);
+    results.push({
+      file: file.relative,
+      ok: result.ok,
+      errors: result.errors,
+      warnings: result.warnings,
+    });
+    if (!result.ok) totalErrors++;
+  }
+
+  // Summary
+  const summary = {
+    ok: totalErrors === 0,
+    repo,
+    total_md_files: candidates.length,
+    passed: candidates.length - totalErrors,
+    failed: totalErrors,
+    results,
+  };
+
+  console.log(JSON.stringify(summary, null, 2));
+
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+  process.exit(totalErrors === 0 ? 0 : 1);
+}
+
+main().catch(e => {
+  console.error(JSON.stringify({ ok: false, errors: [e.message] }));
+  process.exit(1);
+});