promptgraph-mcp 2.4.8 → 2.6.0

package/api.js

@@ -39,7 +39,8 @@ export async function index(sourceDir, sourceName) {
     const path = await import('path');
     const { createHash } = await import('crypto');
     const { parseSkillFile, isSkillFile } = await import('./parser.js');
-    const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
+    const { embedBatch } = await import('./embedder.js');
+    const { BATCH_SIZE } = await import('./config.js');
     const { skillId, vecToBlob } = await import('./db.js');
     const { chunkText } = await import('./chunker.js');
 
@@ -105,7 +106,8 @@ export async function update() {
     const path = await import('path');
     const { createHash } = await import('crypto');
     const { parseSkillFile, isSkillFile } = await import('./parser.js');
-    const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
+    const { embedBatch } = await import('./embedder.js');
+    const { BATCH_SIZE } = await import('./config.js');
     const { skillId, vecToBlob } = await import('./db.js');
     const { chunkText } = await import('./chunker.js');
     const { indexBatch } = await import('./indexer.js');

package/bundle-counts.js

@@ -20,12 +20,13 @@ function httpsGet(url) {
   const headers = { 'User-Agent': 'promptgraph-mcp' };
   if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
   return new Promise((res, rej) => {
-    const req = https.get(url, { headers }, r => {
+    const req = https.get(url, { headers, timeout: 15000 }, r => {
       if (r.statusCode === 403 || r.statusCode === 429) return rej(new Error(`Rate limited`));
       if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
-      let d = ''; r.setEncoding('utf8'); r.on('data', c => d += c); r.on('end', () => res(d));
+      const chunks = []; r.setEncoding('utf8'); r.on('data', c => chunks.push(c)); r.on('end', () => res(chunks.join('')));
     });
     req.on('error', rej);
+    req.on('timeout', () => { req.destroy(new Error('timeout')); });
   });
 }
 

package/config.js

@@ -8,6 +8,13 @@ export const PROMPTGRAPH_DIR = path.join(CLAUDE_DIR, '.promptgraph');
 export const SKILLS_STORE_DIR = path.join(CLAUDE_DIR, 'skills-store');
 const CONFIG_PATH = path.join(PROMPTGRAPH_DIR, 'config.json');
 
+export const MAX_DOWNLOAD_SIZE = 50 * 1024 * 1024  // 50 MB per file
+export const MAX_FILE_COUNT = 50000                  // max files per repo
+export const MAX_REPO_SIZE = 500 * 1024 * 1024      // 500 MB per repo
+export const RATE_LIMIT_REQUESTS = 30               // requests per window
+export const RATE_LIMIT_WINDOW_MS = 60000           // 1 minute window
+export const BATCH_SIZE = 100                       // batch indexing size
+
 const DEFAULTS = {
   sources: [
     { dir: path.join(CLAUDE_DIR, 'skills-store'), source: 'skills-store' },

package/db.js

@@ -7,96 +7,138 @@ const DB_PATH = path.join(PROMPTGRAPH_DIR, 'promptgraph.db');
 
 let _db = null;
 
-export function getDb() {
-  if (_db) return _db;
-  fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
-  const db = new Database(DB_PATH);
-  _db = db;
-  db.pragma('journal_mode = WAL');
-
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS skills (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      description TEXT,
-      path TEXT NOT NULL,
-      source TEXT NOT NULL,
-      content TEXT NOT NULL,
-      hash TEXT
-    );
-
-    CREATE TABLE IF NOT EXISTS chunks (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      skill_id TEXT NOT NULL,
-      chunk_index INTEGER NOT NULL,
-      text TEXT NOT NULL,
-      embedding BLOB NOT NULL,
-      UNIQUE(skill_id, chunk_index)
-    );
-
-    CREATE TABLE IF NOT EXISTS edges (
-      from_skill TEXT NOT NULL,
-      to_skill TEXT NOT NULL,
-      PRIMARY KEY (from_skill, to_skill)
-    );
-
-    CREATE TABLE IF NOT EXISTS ratings (
-      skill_id TEXT PRIMARY KEY,
-      uses INTEGER DEFAULT 0,
-      success INTEGER DEFAULT 0,
-      fail INTEGER DEFAULT 0
-    );
-
-    CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
-      id UNINDEXED,
-      name,
-      description,
-      content,
-      content='skills',
-      content_rowid='rowid'
-    );
-  `);
+const MIGRATIONS = [
+  {
+    version: 1, description: 'initial schema',
+    up: db => db.exec(`
+      CREATE TABLE IF NOT EXISTS skills (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        description TEXT,
+        path TEXT NOT NULL,
+        source TEXT NOT NULL,
+        content TEXT NOT NULL,
+        hash TEXT
+      );
+      CREATE TABLE IF NOT EXISTS chunks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        skill_id TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        text TEXT NOT NULL,
+        embedding BLOB NOT NULL,
+        UNIQUE(skill_id, chunk_index)
+      );
+      CREATE TABLE IF NOT EXISTS edges (
+        from_skill TEXT NOT NULL,
+        to_skill TEXT NOT NULL,
+        PRIMARY KEY (from_skill, to_skill)
+      );
+      CREATE TABLE IF NOT EXISTS ratings (
+        skill_id TEXT PRIMARY KEY,
+        uses INTEGER DEFAULT 0,
+        success INTEGER DEFAULT 0,
+        fail INTEGER DEFAULT 0
+      );
+      CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
+        id UNINDEXED, name, description, content,
+        content='skills', content_rowid='rowid'
+      );
+    `),
+  },
+  {
+    version: 2, description: 'add hash column',
+    up: db => {
+      const cols = db.pragma('table_info(skills)').map(c => c.name);
+      if (!cols.includes('hash')) db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
+    },
+  },
+  {
+    version: 3, description: 'add registry metadata columns',
+    up: db => {
+      const cols = db.pragma('table_info(skills)').map(c => c.name);
+      for (const [col, def] of [
+        ['version', 'TEXT'],
+        ['author', 'TEXT'],
+        ['license', 'TEXT'],
+        ['updated_at', 'TEXT'],
+        ['downloads', 'INTEGER DEFAULT 0'],
+        ['verified', 'INTEGER DEFAULT 0'],
+        ['trust_level', "TEXT DEFAULT 'unknown'"],
+        ['rating', 'REAL DEFAULT 0'],
+        ['rating_count', 'INTEGER DEFAULT 0'],
+        ['popularity', 'REAL DEFAULT 0'],
+        ['last_update', 'TEXT'],
+      ]) {
+        if (!cols.includes(col)) db.exec(`ALTER TABLE skills ADD COLUMN ${col} ${def}`);
+      }
+    },
+  },
+  {
+    version: 4, description: 'registry_entries table',
+    up: db => db.exec(`
+      CREATE TABLE IF NOT EXISTS registry_entries (
+        id TEXT PRIMARY KEY,
+        trust_level TEXT DEFAULT 'unknown',
+        downloads INTEGER DEFAULT 0,
+        rating REAL DEFAULT 0,
+        rating_count INTEGER DEFAULT 0,
+        popularity REAL DEFAULT 0,
+        last_update TEXT
+      );
+    `),
+  },
+  {
+    version: 5, description: 'convert TEXT embeddings to Float32 BLOB',
+    up: db => {
+      const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
+      if (textEmbeddings?.n > 0) {
+        const rows = db.prepare("SELECT rowid, embedding FROM chunks WHERE typeof(embedding) = 'text'").all();
+        const upd = db.prepare('UPDATE chunks SET embedding = ? WHERE rowid = ?');
+        db.transaction(() => {
+          for (const row of rows) {
+            const vec = JSON.parse(row.embedding);
+            upd.run(Buffer.from(new Float32Array(vec).buffer), row.rowid);
+          }
+        })();
+        console.error(`[PromptGraph] Migrated ${textEmbeddings.n} embeddings TEXT→BLOB`);
+      }
+    },
+  },
+]
 
-  // migrate: add hash column if missing
-  const cols = db.pragma('table_info(skills)').map(c => c.name);
-  if (!cols.includes('hash')) {
-    db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
+function getSchemaVersion(db) {
+  try {
+    return db.prepare('SELECT MAX(version) as v FROM _schema_version').get().v || 0
+  } catch {
+    return 0
   }
+}
 
-  // migrate: add registry metadata columns
-  if (!cols.includes('version')) {
-    db.exec('ALTER TABLE skills ADD COLUMN version TEXT');
-  }
-  if (!cols.includes('author')) {
-    db.exec('ALTER TABLE skills ADD COLUMN author TEXT');
-  }
-  if (!cols.includes('license')) {
-    db.exec('ALTER TABLE skills ADD COLUMN license TEXT');
-  }
-  if (!cols.includes('updated_at')) {
-    db.exec('ALTER TABLE skills ADD COLUMN updated_at TEXT');
-  }
-  if (!cols.includes('downloads')) {
-    db.exec('ALTER TABLE skills ADD COLUMN downloads INTEGER DEFAULT 0');
-  }
-  if (!cols.includes('verified')) {
-    db.exec('ALTER TABLE skills ADD COLUMN verified INTEGER DEFAULT 0');
-  }
+function runMigrations(db) {
+  const current = getSchemaVersion(db)
+  const pending = MIGRATIONS.filter(m => m.version > current).sort((a, b) => a.version - b.version)
+  if (!pending.length) return
+
+  db.exec(`CREATE TABLE IF NOT EXISTS _schema_version (version INTEGER PRIMARY KEY, description TEXT, applied_at TEXT)`)
 
-  // migrate: convert JSON text embeddings to Float32 BLOB (one-time, ~10x smaller)
-  const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
-  if (textEmbeddings?.n > 0) {
-    const rows = db.prepare("SELECT rowid, embedding FROM chunks WHERE typeof(embedding) = 'text'").all();
-    const upd = db.prepare('UPDATE chunks SET embedding = ? WHERE rowid = ?');
+  for (const migration of pending) {
     db.transaction(() => {
-      for (const row of rows) {
-        const vec = JSON.parse(row.embedding);
-        upd.run(Buffer.from(new Float32Array(vec).buffer), row.rowid);
-      }
-    })();
-    console.error(`[PromptGraph] Migrated ${textEmbeddings.n} embeddings TEXT→BLOB`);
+      migration.up(db)
+      db.prepare('INSERT INTO _schema_version (version, description, applied_at) VALUES (?, ?, ?)').run(
+        migration.version, migration.description, new Date().toISOString()
+      )
+    })()
+    console.error(`[PromptGraph] DB migrated to v${migration.version}: ${migration.description}`)
   }
+}
 
+export function getDb() {
+  if (_db) return _db;
+  fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
+  const db = new Database(DB_PATH);
+  _db = db;
+  db.pragma('journal_mode = WAL');
+  runMigrations(db);
   return db;
 }
 

package/github-import.js

@@ -4,24 +4,71 @@ import fs from 'fs';
 import https from 'https';
 import { globSync } from 'glob';
 import { indexAll, indexSource } from './indexer.js';
-import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR } from './config.js';
+import { loadConfig, saveConfig, PROMPTGRAPH_DIR, SKILLS_STORE_DIR, MAX_DOWNLOAD_SIZE, MAX_FILE_COUNT, MAX_REPO_SIZE, RATE_LIMIT_REQUESTS, RATE_LIMIT_WINDOW_MS } from './config.js';
 import { validateSkill } from './validator.js';
 import { isSkillFile } from './parser.js';
+import { RateLimiter } from './src/utils/rate-limiter.js';
+
+const githubRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS, windowMs: RATE_LIMIT_WINDOW_MS })
+const downloadRateLimiter = new RateLimiter({ maxRequests: RATE_LIMIT_REQUESTS * 2, windowMs: RATE_LIMIT_WINDOW_MS })
 
 const SKILL_DIRS = ['skills', 'commands', 'prompts', 'agents', 'skills-store', 'slash-commands', 'custom-commands', 'templates'];
 
 // ── helpers ───────────────────────────────────────────────────────────────────
 
-function httpsGet(url) {
+const repoStats = new Map()
+
+function getRepoStats(ownerRepo) {
+  if (!repoStats.has(ownerRepo)) {
+    repoStats.set(ownerRepo, { totalBytes: 0, fileCount: 0 })
+  }
+  return repoStats.get(ownerRepo)
+}
+
+function streamDownload(url, maxSize = MAX_DOWNLOAD_SIZE, redirects = 0) {
+  if (redirects > 5) return Promise.reject(new Error('Too many redirects'))
+  return new Promise((res, rej) => {
+    const token = process.env.GITHUB_TOKEN;
+    const headers = { 'User-Agent': 'promptgraph-mcp' };
+    if (token && url.startsWith('https://raw.')) headers['Authorization'] = `Bearer ${token}`;
+    const req = https.get(url, { headers }, r => {
+      if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location)
+        return streamDownload(r.headers.location, maxSize, redirects + 1).then(res, rej);
+      if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
+      const cl = parseInt(r.headers['content-length'], 10);
+      if (!isNaN(cl) && cl > maxSize) {
+        r.resume();
+        return rej(new Error(`Content-Length ${cl} exceeds max ${maxSize}`));
+      }
+      const chunks = []
+      let total = 0
+      r.setEncoding('utf8')
+      r.on('data', c => {
+        total += Buffer.byteLength(c, 'utf8')
+        if (total > maxSize) {
+          r.destroy()
+          return rej(new Error(`Download exceeded ${maxSize} bytes`))
+        }
+        chunks.push(c)
+      })
+      r.on('end', () => res(chunks.join('')))
+    })
+    req.on('error', rej)
+  })
+}
+
+async function httpsGet(url, redirects = 0) {
+  if (redirects > 5) return Promise.reject(new Error('Too many redirects'))
+  await githubRateLimiter.acquire()
   const token = process.env.GITHUB_TOKEN;
   const headers = { 'User-Agent': 'promptgraph-mcp' };
   if (token && url.startsWith('https://api.github.com/')) headers['Authorization'] = `Bearer ${token}`;
   return new Promise((res, rej) => {
     const req = https.get(url, { headers }, r => {
       if (r.statusCode >= 300 && r.statusCode < 400 && r.headers.location)
-        return httpsGet(r.headers.location).then(res, rej);
+        return httpsGet(r.headers.location, redirects + 1).then(res, rej);
       if (r.statusCode !== 200) { r.resume(); return rej(new Error(`HTTP ${r.statusCode}`)); }
-      let d = ''; r.setEncoding('utf8'); r.on('data', c => d += c); r.on('end', () => res(d));
+      const chunks = []; r.setEncoding('utf8'); r.on('data', c => chunks.push(c)); r.on('end', () => res(chunks.join('')));
     });
     req.on('error', rej);
   });
@@ -39,11 +86,22 @@ function repoExists(ownerRepo) {
 }
 
 // Download one .md file, run validateSkill on it, return errors/warnings.
-async function validateMdFile(file, tmpDir) {
+async function validateMdFile(file, tmpDir, ownerRepo) {
   const errors = [];
   const warnings = [];
+  const stats = getRepoStats(ownerRepo);
   try {
-    const content = await httpsGet(file.download_url);
+    if (stats.fileCount >= MAX_FILE_COUNT) {
+      errors.push(`${file.name}: skipped — repo file count limit (${MAX_FILE_COUNT}) reached`);
+      return { errors, warnings };
+    }
+    await downloadRateLimiter.acquire()
+    const content = await streamDownload(file.download_url);
+    stats.totalBytes += Buffer.byteLength(content, 'utf8');
+    stats.fileCount++;
+    if (stats.totalBytes > MAX_REPO_SIZE) {
+      return { errors: [...errors, `${file.name}: repo size limit (${MAX_REPO_SIZE}) exceeded`], warnings };
+    }
     const tmpPath = path.join(tmpDir, file.name);
     fs.mkdirSync(path.dirname(tmpPath), { recursive: true });
     fs.writeFileSync(tmpPath, content);
@@ -119,7 +177,7 @@ export async function validateRepoSkills(ownerRepo) {
   let warnings = [];
 
   for (const file of mdToValidate) {
-    const r = await validateMdFile(file, tmpDir);
+    const r = await validateMdFile(file, tmpDir, ownerRepo);
     errors.push(...r.errors);
     warnings.push(...r.warnings);
   }
@@ -464,6 +522,10 @@ export async function importFromGitHub(repoUrl) {
   const label = skillsSubdir || localLabel;
   const mdFiles = globSync(`${skillsDir}/**/*.md`);
 
+  if (mdFiles.length > MAX_FILE_COUNT) {
+    console.warn(`Warning: ${mdFiles.length} .md files exceeds limit of ${MAX_FILE_COUNT} — truncating`);
+  }
+
   if (skillsSubdir) {
     console.log(`Sparse-checkout: ${label}/ only (${mdFiles.length} .md files, no other repo files)`);
   } else {