promptgraph-mcp 2.4.7 → 2.5.0

package/src/store/hnsw-store.js

@@ -0,0 +1,187 @@
+import { VectorStore } from './vector-store.js';
+import fs from 'fs';
+import path from 'path';
+
+const DIMS = 384;
+const EF_SEARCH = 100;
+
+let HierarchicalNSW = null;
+let moduleError = null;
+
+async function getHnswModule() {
+  if (HierarchicalNSW) return HierarchicalNSW;
+  if (moduleError) throw moduleError;
+  try {
+    const mod = await import('hnswlib-node');
+    HierarchicalNSW = mod.default.HierarchicalNSW;
+    return HierarchicalNSW;
+  } catch (e) {
+    moduleError = new Error(
+      'hnswlib-node not available. Run: npm install hnswlib-node'
+    );
+    throw moduleError;
+  }
+}
+
+function toArray(v) {
+  return Array.from(v);
+}
+
+export class HNSWVectorStore extends VectorStore {
+  constructor() {
+    super();
+    this._index = null;
+    this._vectors = [];
+    this._idMap = [];
+    this._itemCount = 0;
+    this._maxElements = 0;
+  }
+
+  async add(id, vector) {
+    const HNSW = await getHnswModule();
+    const vec = toArray(vector);
+    if (!this._index) {
+      this._maxElements = 10000;
+      this._index = new HNSW('cosine', DIMS);
+      this._index.initIndex(this._maxElements, 200);
+    }
+    if (this._itemCount >= this._maxElements) {
+      await this._rebuild(this._maxElements * 2);
+    }
+    this._index.addPoint(vec, this._itemCount);
+    this._vectors.push(vec);
+    this._idMap.push(id);
+    this._itemCount++;
+  }
+
+  async addBatch(entries) {
+    for (const { id, vector } of entries) {
+      await this.add(id, vector);
+    }
+  }
+
+  async remove(id) {
+    const remainingVectors = [];
+    const remainingIds = [];
+    for (let i = 0; i < this._itemCount; i++) {
+      if (this._idMap[i] !== id) {
+        remainingVectors.push(this._vectors[i]);
+        remainingIds.push(this._idMap[i]);
+      }
+    }
+    this._index = null;
+    this._vectors = remainingVectors;
+    this._idMap = remainingIds;
+    this._itemCount = remainingVectors.length;
+    this._maxElements = 0;
+    if (this._itemCount > 0) {
+      await this._rebuild(this._itemCount + 1000);
+    }
+  }
+
+  async search(vector, topK = 20) {
+    await getHnswModule();
+    if (!this._index || this._itemCount === 0) return [];
+
+    const efSearch = Math.max(topK * 4, EF_SEARCH);
+    this._index.setEf(efSearch);
+
+    const numCandidates = Math.min(topK * 8, this._itemCount);
+    const result = this._index.searchKnn(toArray(vector), numCandidates);
+
+    const bestBySkill = new Map();
+    const neighbors = result.neighbors;
+    const distances = result.distances;
+    for (let i = 0; i < neighbors.length; i++) {
+      const idx = neighbors[i];
+      const score = 1 - distances[i];
+      const skillId = this._idMap[idx];
+      const prev = bestBySkill.get(skillId);
+      if (!prev || score > prev) bestBySkill.set(skillId, score);
+    }
+
+    return [...bestBySkill.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, topK)
+      .map(([skill_id, score]) => ({ skill_id, score }));
+  }
+
+  async build(entries) {
+    const HNSW = await getHnswModule();
+    const count = entries.length;
+
+    this._vectors = [];
+    this._idMap = [];
+    this._itemCount = 0;
+    this._maxElements = 0;
+    this._index = null;
+
+    if (count === 0) return;
+
+    this._maxElements = count + 1000;
+    this._index = new HNSW('cosine', DIMS);
+    this._index.initIndex(this._maxElements, 200);
+
+    for (let i = 0; i < count; i++) {
+      const { skill_id, vector } = entries[i];
+      const vec = toArray(vector);
+      this._index.addPoint(vec, i);
+      this._vectors.push(vec);
+      this._idMap.push(skill_id);
+      this._itemCount++;
+    }
+  }
+
+  async clear() {
+    this._index = null;
+    this._vectors = [];
+    this._idMap = [];
+    this._itemCount = 0;
+    this._maxElements = 0;
+  }
+
+  get size() {
+    return this._itemCount;
+  }
+
+  async save(dir) {
+    const HNSW = await getHnswModule();
+    if (!this._index) return;
+    fs.mkdirSync(dir, { recursive: true });
+    this._index.writeIndexSync(path.join(dir, 'index.bin'));
+    fs.writeFileSync(path.join(dir, 'vectors.json'), JSON.stringify(this._vectors), 'utf8');
+    fs.writeFileSync(path.join(dir, 'idmap.json'), JSON.stringify(this._idMap), 'utf8');
+  }
+
+  async load(dir) {
+    const HNSW = await getHnswModule();
+    const indexPath = path.join(dir, 'index.bin');
+    const vectorsPath = path.join(dir, 'vectors.json');
+    const idmapPath = path.join(dir, 'idmap.json');
+    if (!fs.existsSync(indexPath)) return false;
+
+    this._index = new HNSW('cosine', DIMS);
+    this._index.readIndexSync(indexPath);
+    this._vectors = JSON.parse(fs.readFileSync(vectorsPath, 'utf8'));
+    this._idMap = JSON.parse(fs.readFileSync(idmapPath, 'utf8'));
+    this._itemCount = this._idMap.length;
+    this._maxElements = this._index.getMaxElements();
+    return true;
+  }
+
+  static async fromDir(dir) {
+    const store = new HNSWVectorStore();
+    await store.load(dir);
+    return store;
+  }
+
+  async _rebuild(newMax) {
+    const HNSW = await getHnswModule();
+    this._maxElements = newMax;
+    this._index = new HNSW('cosine', DIMS);
+    this._index.initIndex(this._maxElements, 200);
+    for (let i = 0; i < this._itemCount; i++) {
+      this._index.addPoint(this._vectors[i], i);
+    }
+  }
+}

package/src/store/index.js

@@ -0,0 +1,19 @@
+import { FlatVectorStore } from './flat-store.js';
+import { HNSWVectorStore } from './hnsw-store.js';
+
+let _store = null;
+
+export function getStore(type = null) {
+  if (_store) return _store;
+
+  if (!type) {
+    type = process.env.PG_VECTOR_STORE || 'flat';
+  }
+
+  _store = type === 'hnsw' ? new HNSWVectorStore() : new FlatVectorStore();
+  return _store;
+}
+
+export function resetStore() {
+  _store = null;
+}

package/src/store/vector-store.js

@@ -0,0 +1,9 @@
+export class VectorStore {
+  async add(id, vector) { throw new Error('not implemented'); }
+  async addBatch(entries) { throw new Error('not implemented'); }
+  async remove(id) { throw new Error('not implemented'); }
+  async search(vector, topK) { throw new Error('not implemented'); }
+  async build(entries) { throw new Error('not implemented'); }
+  async clear() { throw new Error('not implemented'); }
+  get size() { return 0; }
+}

package/src/utils/rate-limiter.js

@@ -0,0 +1,33 @@
+export class RateLimiter {
+  constructor({ maxRequests, windowMs }) {
+    this.maxRequests = maxRequests
+    this.windowMs = windowMs
+    this.timestamps = []
+    this._chain = Promise.resolve()
+  }
+
+  async acquire() {
+    const prev = this._chain
+    this._chain = prev.then(() => this._doAcquire())
+    return this._chain
+  }
+
+  async _doAcquire() {
+    while (!this.tryAcquire()) {
+      const oldest = this.timestamps[0]
+      const waitMs = oldest
+        ? Math.max(1, oldest + this.windowMs - Date.now())
+        : 100
+      await new Promise(r => setTimeout(r, Math.min(waitMs, this.windowMs)))
+    }
+  }
+
+  tryAcquire() {
+    const now = Date.now()
+    const cutoff = now - this.windowMs
+    this.timestamps = this.timestamps.filter(t => t > cutoff)
+    if (this.timestamps.length >= this.maxRequests) return false
+    this.timestamps.push(now)
+    return true
+  }
+}

package/validator.js

@@ -1,5 +1,7 @@
 import fs from 'fs';
+import path from 'path';
 import matter from 'gray-matter';
+import { MAX_DOWNLOAD_SIZE } from './config.js';
 
 // patterns that indicate malicious or junk skills
 const DANGEROUS_PATTERNS = [
@@ -15,7 +17,7 @@ const DANGEROUS_PATTERNS = [
 ];
 
 const MIN_CONTENT_LENGTH = 200;       // chars of actual instruction
-const MAX_CONTENT_LENGTH = 100000;    // 100KB cap
+const MAX_CONTENT_LENGTH = 5242880;    // 5MB cap
 const MIN_DESCRIPTION_LENGTH = 15;
 const NAME_RE = /^[a-z0-9][a-z0-9-]{1,63}$/;
 
@@ -80,6 +82,25 @@ export function validateSkill(filePath) {
 
   // junk filename heuristic
   const base = filePath.split(/[\\/]/).pop().toLowerCase();
+
+  // path traversal check (must be an actual path component, not substring match)
+  const pathParts = filePath.split(/[\\/]/);
+  if (pathParts.includes('..')) {
+    errors.push('Path traversal detected: file path contains ".."');
+  }
+
+  // extension whitelist — reject unexpected file types
+  const ALLOWED_EXTENSIONS = new Set(['.md', '.json', '.yaml', '.yml', '.txt', '.js', '.ts', '.py', '.rs', '.toml']);
+  const ext = path.extname(filePath).toLowerCase();
+  if (!ALLOWED_EXTENSIONS.has(ext)) {
+    errors.push(`File extension "${ext}" not allowed. Allowed: ${[...ALLOWED_EXTENSIONS].join(', ')}`);
+  }
+
+  // binary content detection — must be valid UTF-8 with no null bytes
+  if (raw.includes('\0')) {
+    errors.push('File contains null bytes (binary content)');
+  }
+
   if (['readme.md', 'changelog.md', 'license.md', 'contributing.md'].includes(base)) {
     warnings.push('Filename looks like a docs file, not a skill.');
   }
@@ -136,6 +157,15 @@ export function validateBundle(def) {
   return { ok: errors.length === 0, errors, warnings };
 }
 
+export function sanitizeExternalContent(content) {
+  if (typeof content !== 'string') return ''
+  let sanitized = content.replace(/\0/g, '')
+  if (Buffer.byteLength(sanitized, 'utf8') > MAX_DOWNLOAD_SIZE) {
+    sanitized = Buffer.from(sanitized, 'utf8').subarray(0, MAX_DOWNLOAD_SIZE).toString('utf8')
+  }
+  return sanitized
+}
+
 // CLI: node validator.js <file>
 if (import.meta.url === `file://${process.argv[1]}`) {
   const file = process.argv[2];

package/registry/training/bad/accepts-HISTORY.md

@@ -1,250 +0,0 @@
-2.0.0 / 2024-08-31
-==================
-
-  * Drop node <18 support
-  * deps: mime-types@^3.0.0
-  * deps: negotiator@^1.0.0
-
-1.3.8 / 2022-02-02
-==================
-
-  * deps: mime-types@~2.1.34
-    - deps: mime-db@~1.51.0
-  * deps: negotiator@0.6.3
-
-1.3.7 / 2019-04-29
-==================
-
-  * deps: negotiator@0.6.2
-    - Fix sorting charset, encoding, and language with extra parameters
-
-1.3.6 / 2019-04-28
-==================
-
-  * deps: mime-types@~2.1.24
-    - deps: mime-db@~1.40.0
-
-1.3.5 / 2018-02-28
-==================
-
-  * deps: mime-types@~2.1.18
-    - deps: mime-db@~1.33.0
-
-1.3.4 / 2017-08-22
-==================
-
-  * deps: mime-types@~2.1.16
-    - deps: mime-db@~1.29.0
-
-1.3.3 / 2016-05-02
-==================
-
-  * deps: mime-types@~2.1.11
-    - deps: mime-db@~1.23.0
-  * deps: negotiator@0.6.1
-    - perf: improve `Accept` parsing speed
-    - perf: improve `Accept-Charset` parsing speed
-    - perf: improve `Accept-Encoding` parsing speed
-    - perf: improve `Accept-Language` parsing speed
-
-1.3.2 / 2016-03-08
-==================
-
-  * deps: mime-types@~2.1.10
-    - Fix extension of `application/dash+xml`
-    - Update primary extension for `audio/mp4`
-    - deps: mime-db@~1.22.0
-
-1.3.1 / 2016-01-19
-==================
-
-  * deps: mime-types@~2.1.9
-    - deps: mime-db@~1.21.0
-
-1.3.0 / 2015-09-29
-==================
-
-  * deps: mime-types@~2.1.7
-    - deps: mime-db@~1.19.0
-  * deps: negotiator@0.6.0
-    - Fix including type extensions in parameters in `Accept` parsing
-    - Fix parsing `Accept` parameters with quoted equals
-    - Fix parsing `Accept` parameters with quoted semicolons
-    - Lazy-load modules from main entry point
-    - perf: delay type concatenation until needed
-    - perf: enable strict mode
-    - perf: hoist regular expressions
-    - perf: remove closures getting spec properties
-    - perf: remove a closure from media type parsing
-    - perf: remove property delete from media type parsing
-
-1.2.13 / 2015-09-06
-===================
-
-  * deps: mime-types@~2.1.6
-    - deps: mime-db@~1.18.0
-
-1.2.12 / 2015-07-30
-===================
-
-  * deps: mime-types@~2.1.4
-    - deps: mime-db@~1.16.0
-
-1.2.11 / 2015-07-16
-===================
-
-  * deps: mime-types@~2.1.3
-    - deps: mime-db@~1.15.0
-
-1.2.10 / 2015-07-01
-===================
-
-  * deps: mime-types@~2.1.2
-    - deps: mime-db@~1.14.0
-
-1.2.9 / 2015-06-08
-==================
-
-  * deps: mime-types@~2.1.1
-    - perf: fix deopt during mapping
-
-1.2.8 / 2015-06-07
-==================
-
-  * deps: mime-types@~2.1.0
-    - deps: mime-db@~1.13.0
-  * perf: avoid argument reassignment & argument slice
-  * perf: avoid negotiator recursive construction
-  * perf: enable strict mode
-  * perf: remove unnecessary bitwise operator
-
-1.2.7 / 2015-05-10
-==================
-
-  * deps: negotiator@0.5.3
-    - Fix media type parameter matching to be case-insensitive
-
-1.2.6 / 2015-05-07
-==================
-
-  * deps: mime-types@~2.0.11
-    - deps: mime-db@~1.9.1
-  * deps: negotiator@0.5.2
-    - Fix comparing media types with quoted values
-    - Fix splitting media types with quoted commas
-
-1.2.5 / 2015-03-13
-==================
-
-  * deps: mime-types@~2.0.10
-    - deps: mime-db@~1.8.0
-
-1.2.4 / 2015-02-14
-==================
-
-  * Support Node.js 0.6
-  * deps: mime-types@~2.0.9
-    - deps: mime-db@~1.7.0
-  * deps: negotiator@0.5.1
-    - Fix preference sorting to be stable for long acceptable lists
-
-1.2.3 / 2015-01-31
-==================
-
-  * deps: mime-types@~2.0.8
-    - deps: mime-db@~1.6.0
-
-1.2.2 / 2014-12-30
-==================
-
-  * deps: mime-types@~2.0.7
-    - deps: mime-db@~1.5.0
-
-1.2.1 / 2014-12-30
-==================
-
-  * deps: mime-types@~2.0.5
-    - deps: mime-db@~1.3.1
-
-1.2.0 / 2014-12-19
-==================
-
-  * deps: negotiator@0.5.0
-    - Fix list return order when large accepted list
-    - Fix missing identity encoding when q=0 exists
-    - Remove dynamic building of Negotiator class
-
-1.1.4 / 2014-12-10
-==================
-
-  * deps: mime-types@~2.0.4
-    - deps: mime-db@~1.3.0
-
-1.1.3 / 2014-11-09
-==================
-
-  * deps: mime-types@~2.0.3
-    - deps: mime-db@~1.2.0
-
-1.1.2 / 2014-10-14
-==================
-
-  * deps: negotiator@0.4.9
-    - Fix error when media type has invalid parameter
-
-1.1.1 / 2014-09-28
-==================
-
-  * deps: mime-types@~2.0.2
-    - deps: mime-db@~1.1.0
-  * deps: negotiator@0.4.8
-    - Fix all negotiations to be case-insensitive
-    - Stable sort preferences of same quality according to client order
-
-1.1.0 / 2014-09-02
-==================
-
-  * update `mime-types`
-
-1.0.7 / 2014-07-04
-==================
-
-  * Fix wrong type returned from `type` when match after unknown extension
-
-1.0.6 / 2014-06-24
-==================
-
-  * deps: negotiator@0.4.7
-
-1.0.5 / 2014-06-20
-==================
-
- * fix crash when unknown extension given
-
-1.0.4 / 2014-06-19
-==================
-
-  * use `mime-types`
-
-1.0.3 / 2014-06-11
-==================
-
-  * deps: negotiator@0.4.6
-    - Order by specificity when quality is the same
-
-1.0.2 / 2014-05-29
-==================
-
-  * Fix interpretation when header not in request
-  * deps: pin negotiator@0.4.5
-
-1.0.1 / 2014-01-18
-==================
-
-  * Identity encoding isn't always acceptable
-  * deps: negotiator@~0.4.0
-
-1.0.0 / 2013-12-27
-==================
-
-  * Genesis