promptgraph-mcp 2.9.62 → 2.10.0

package/README.md

@@ -77,6 +77,10 @@ pg update                       # Update promptgraph-mcp to latest version
 pg search <query>               # Semantic search across your skills
 pg marketplace                  # Browse community skill bundles (TUI)
 
+# Skill graph
+pg plan <skill>                 # DAG execution plan: order, parallel levels, cycle detection
+pg duplicates                   # Find near-duplicate / overlapping skills
+
 # Install
 pg install <id>                 # Install a skill by marketplace ID (pg-xxxxxx)
 pg import <github-url>          # Import skills directly from a GitHub repo URL

package/commands/duplicates.js

@@ -0,0 +1,41 @@
+import { success, error, info } from '../cli.js';
+import chalk from 'chalk';
+
+// pg duplicates [--threshold 0.92] [--json]  — flag near-duplicate / overlapping skills.
+export default async function handler(args, bin) {
+  const ti = args.indexOf('--threshold');
+  const threshold = ti !== -1 && args[ti + 1] ? parseFloat(args[ti + 1]) : 0.70;
+  const asJson = args.includes('--json');
+
+  const spin = (await import('../cli.js')).spinner('Comparing skill embeddings...');
+  spin.start();
+  let pairs;
+  try {
+    const { findDuplicates } = await import('../duplicates.js');
+    pairs = await findDuplicates({ threshold });
+  } catch (e) {
+    spin.stop();
+    error(`Duplicate scan failed: ${e.message}`);
+    process.exit(1);
+  }
+  spin.stop();
+
+  if (asJson) { console.log(JSON.stringify(pairs, null, 2)); process.exit(0); }
+
+  if (!pairs.length) {
+    success(`No near-duplicate skills found above similarity ${threshold}.`);
+    process.exit(0);
+  }
+
+  console.log('\n' + chalk.bold(`${pairs.length} near-duplicate pair${pairs.length === 1 ? '' : 's'}`) +
+    chalk.gray(`  (cosine ≥ ${threshold})\n`));
+  for (const p of pairs) {
+    const simColor = p.sim >= 0.73 ? chalk.red : p.sim >= 0.71 ? chalk.yellow : chalk.gray;
+    const cross = p.sameSource ? '' : chalk.gray('  (different sources)');
+    console.log(`  ${simColor(p.sim.toFixed(3))}  ${chalk.cyan(p.aName)} ${chalk.gray('≈')} ${chalk.cyan(p.bName)}${cross}`);
+    console.log(`         ${chalk.gray(p.a)}`);
+    console.log(`         ${chalk.gray(p.b)}`);
+  }
+  console.log(chalk.gray('\n  Review pairs ≥ 0.73 first — likely true duplicates worth merging or removing.\n'));
+  process.exit(0);
+}

package/commands/plan.js

@@ -0,0 +1,45 @@
+import { success, error, info } from '../cli.js';
+import chalk from 'chalk';
+
+// pg plan <skill>  — show the DAG execution plan for a goal skill.
+export default async function handler(args, bin) {
+  const target = args.slice(1).filter(a => !a.startsWith('-')).join(' ');
+  if (!target) { error('Usage: pg plan <skill-id-or-name>'); process.exit(1); }
+  const asJson = args.includes('--json');
+
+  const { buildPlan } = await import('../planner.js');
+  const plan = buildPlan(target);
+  if (plan?.error) { error(plan.error); process.exit(1); }
+
+  if (asJson) { console.log(JSON.stringify(plan, null, 2)); process.exit(0); }
+
+  const name = (id) => plan.nodes[id]?.name || id;
+
+  console.log('\n' + chalk.bold(`Plan for ${chalk.cyan(plan.root.name)}`) +
+    chalk.gray(`  (${plan.count} skill${plan.count === 1 ? '' : 's'})`));
+
+  if (!plan.acyclic) {
+    console.log(chalk.red(`\n  ✗ Cycle detected — cannot execute as-is:`));
+    for (const cyc of plan.cycles) {
+      console.log('    ' + chalk.red(cyc.map(name).join(' → ')));
+    }
+    console.log(chalk.gray('    Break the cycle (a skill ends up depending on itself).'));
+  }
+
+  if (plan.unresolved.length) {
+    console.log(chalk.yellow(`\n  ⚠ Referenced but not indexed (${plan.unresolved.length}):`));
+    for (const u of plan.unresolved) console.log('    ' + chalk.yellow(u));
+  }
+
+  if (plan.acyclic && plan.levels.length) {
+    console.log(chalk.bold('\n  Execution order ') + chalk.gray('(dependencies first; each level can run in parallel):'));
+    plan.levels.forEach((batch, i) => {
+      const tag = batch.length > 1 ? chalk.gray(` (${batch.length} parallel)`) : '';
+      console.log(`    ${chalk.gray(`L${i}`)}  ${batch.map(id => chalk.white(name(id))).join(chalk.gray(' · '))}${tag}`);
+    });
+    console.log(chalk.gray('\n  Linear order: ') + plan.order.map(name).join(chalk.gray(' → ')));
+  }
+
+  console.log();
+  process.exit(0);
+}

package/duplicates.js

@@ -0,0 +1,70 @@
+import { getDb, blobToVec } from './db.js';
+import { cosineSimilarity } from './embedder.js';
+import { annSearch, buildAnnIndex } from './ann.js';
+import { getStore } from './src/store/index.js';
+
+// ── Near-duplicate / overlap detector ────────────────────────────────────────
+// With ~2000 skills, overlap and redundancy are real. Represent each skill by the
+// mean of its chunk embeddings, use the ANN index to get candidate neighbors
+// (avoids O(n²)), then confirm with exact cosine on the skill-level vectors.
+
+function skillMeanVectors(db) {
+  const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
+  const acc = new Map();   // skill_id -> { vec: Float32Array, n }
+  for (const r of rows) {
+    const v = blobToVec(r.embedding);
+    const cur = acc.get(r.skill_id);
+    if (!cur) acc.set(r.skill_id, { vec: Float32Array.from(v), n: 1 });
+    else { for (let i = 0; i < v.length; i++) cur.vec[i] += v[i]; cur.n++; }
+  }
+  const out = new Map();
+  for (const [id, { vec, n }] of acc) {
+    for (let i = 0; i < vec.length; i++) vec[i] /= n;
+    out.set(id, vec);
+  }
+  return out;
+}
+
+// Calibrated to BGE-Small mean-pooled vectors, whose similarity scale is compressed:
+// genuine near-duplicates land around 0.70–0.73, not 0.9+. Default 0.70 surfaces real
+// overlap; 0.73+ is near-identical content.
+export async function findDuplicates({ threshold = 0.70, maxPairs = 200, neighbors = 8 } = {}) {
+  const db = getDb();
+  // A fresh CLI process has an empty in-memory ANN store — build it from chunks
+  // so neighbor lookup works (the MCP server keeps it warm; the CLI must not assume that).
+  if (getStore().size === 0) await buildAnnIndex();
+  const vecs = skillMeanVectors(db);
+  const meta = new Map(
+    db.prepare('SELECT id, name, source FROM skills').all().map(s => [s.id, s])
+  );
+
+  const pairs = [];
+  const seen = new Set();
+
+  for (const [id, vec] of vecs) {
+    const hits = await annSearch(vec, neighbors);
+    if (!hits) break;   // ANN unavailable
+    for (const { skill_id: other } of hits) {
+      if (other === id) continue;
+      const key = id < other ? `${id}|${other}` : `${other}|${id}`;
+      if (seen.has(key)) continue;
+      const ov = vecs.get(other);
+      if (!ov) continue;
+      const sim = cosineSimilarity(vec, ov);
+      if (sim >= threshold) {
+        seen.add(key);
+        const a = meta.get(id), b = meta.get(other);
+        pairs.push({
+          a: id, b: other,
+          aName: a?.name || id, bName: b?.name || other,
+          aSource: a?.source || '?', bSource: b?.source || '?',
+          sameSource: a?.source === b?.source,
+          sim: +sim.toFixed(4),
+        });
+      }
+    }
+  }
+
+  pairs.sort((x, y) => y.sim - x.sim);
+  return pairs.slice(0, maxPairs);
+}

package/index.js

@@ -18,7 +18,7 @@ const args = process.argv.slice(2);
 const rawBin = process.argv[1]?.split(/[\\/]/).pop()?.replace(/\.js$/, '');
 const bin = (rawBin && rawBin !== 'index') ? rawBin : 'pg';
 
-const KNOWN_COMMANDS = new Set(['reindex', 'update', 'import', 'install', 'setup', 'validate', 'marketplace', 'doctor', 'search', 'help', '--help', '-h', '--version', '-v', 'version', 'bundle', 'status', 'train', 'add-dir']);
+const KNOWN_COMMANDS = new Set(['reindex', 'update', 'import', 'install', 'setup', 'validate', 'marketplace', 'doctor', 'search', 'help', '--help', '-h', '--version', '-v', 'version', 'bundle', 'status', 'train', 'add-dir', 'plan', 'duplicates']);
 
 function showHelp() {
   console.log(
@@ -34,6 +34,8 @@ function showHelp() {
     ['search <query>',      'Search skills from the terminal'],
     ['import <owner/repo>', 'Import skills from GitHub'],
     ['add-dir <path>',      'Index skills from a local folder (any platform)'],
+    ['plan <skill>',        'Show the DAG execution plan (order, parallel levels, cycles)'],
+    ['duplicates',          'Find near-duplicate / overlapping skills'],
     ['status',              'Show installed skills, repos, and bundles'],
     ['install <name>',      'Install a bundle by name, code, or id'],
     ['marketplace',         'Interactive TUI: browse + search + install skills & bundles'],
@@ -95,6 +97,8 @@ const COMMAND_MAP = {
   update:    './commands/update.js',
   reindex:   './commands/reindex.js',
   'add-dir': './commands/add-dir.js',
+  plan:      './commands/plan.js',
+  duplicates: './commands/duplicates.js',
 }
 
 if (COMMAND_MAP[args[0]]) {
@@ -174,6 +178,15 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
         required: ['name'],
       },
     },
+    {
+      name: 'pg_plan',
+      description: 'Build a DAG execution plan for a goal skill: full transitive dependency graph, topological order (dependencies first), parallelizable levels, cycle detection, and unresolved references. Call before chaining to know the whole plan up front.',
+      inputSchema: {
+        type: 'object',
+        properties: { name: { type: 'string' } },
+        required: ['name'],
+      },
+    },
     {
       name: 'pg_rate',
       description: 'Record skill usage outcome. Call after applying a skill: outcome="success" if it helped, "fail" if it did not.',
@@ -288,6 +301,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       case 'pg_callers': result = getCallers(args.name); break;
       case 'pg_callees': result = getCallees(args.name); break;
       case 'pg_impact': result = getImpact(args.name); break;
+      case 'pg_plan': { const { buildPlan } = await import('./planner.js'); result = buildPlan(args.name); break; }
       case 'pg_rate':
         if (args.outcome === 'use') recordUse(args.skill_id);
         else if (args.outcome === 'success') recordSuccess(args.skill_id);

package/indexer.js

@@ -99,10 +99,10 @@ export async function indexBatch(db, skills, { fast = false, silent = false, onE
     for (const skill of skills) {
       const id = skillId(skill.source, skill.name);
       for (const calledName of skill.calls) {
-        // prefer a skill in the same source, fall back to any, then bare name
-        const same = resolveSameSource.get(calledName, skill.source);
-        const resolved = same || resolveAny.get(calledName);
-        upsertEdge.run(id, resolved ? resolved.id : calledName);
+        // Only keep an edge if it resolves to a REAL indexed skill — otherwise
+        // `/var`, `/usr`, `/scan` (paths & command words) pollute the call graph.
+        const resolved = resolveSameSource.get(calledName, skill.source) || resolveAny.get(calledName);
+        if (resolved && resolved.id !== id) upsertEdge.run(id, resolved.id);
       }
     }
   })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptgraph-mcp",
-  "version": "2.9.62",
+  "version": "2.10.0",
   "files": [
     "*.js",
     "commands/",

package/parser.js

@@ -5,7 +5,9 @@ import { hardFilter } from './src/filter/hard-filter.js';
 import { classify } from './src/filter/classifier.js';
 import { loadModel } from './src/filter/train.js';
 
-const SKILL_REF_RE = /(?<!https?:|ftp:)(?<![a-zA-Z0-9])\/([a-z0-9][a-z0-9-]{2,})/g;
+// /skill-name reference. Trailing (?![a-z0-9-]|\/) forces the full word and rejects
+// path segments like /var/log, /usr/bin (a slash right after the word → not a skill ref).
+const SKILL_REF_RE = /(?<!https?:|ftp:)(?<![a-zA-Z0-9])\/([a-z0-9][a-z0-9-]{2,})(?![a-z0-9-]|\/)/g;
 
 // Generic filenames that carry no identity — name comes from the parent folder.
 const GENERIC_SKILL_FILENAMES = new Set([

package/planner.js

@@ -0,0 +1,132 @@
+import { getDb } from './db.js';
+
+// ── DAG execution planner ─────────────────────────────────────────────────────
+// Skills reference other skills (edge A→B means "A calls/uses B"). This builds the
+// full transitive dependency graph for a goal skill and produces a deterministic
+// execution plan BEFORE anything runs: topological order, parallelizable levels,
+// cycle detection (with the actual cycle path), and dangling/unresolved refs.
+
+function resolveId(db, nameOrId) {
+  const byId = db.prepare('SELECT id FROM skills WHERE id = ?').get(nameOrId);
+  if (byId) return { id: byId.id };
+  const byName = db.prepare('SELECT id FROM skills WHERE name = ?').all(nameOrId);
+  if (byName.length === 1) return { id: byName[0].id };
+  if (byName.length > 1) {
+    return { error: `Ambiguous name "${nameOrId}" — use a full id: ${byName.map(r => r.id).join(', ')}` };
+  }
+  return { error: `Skill not found: ${nameOrId}` };
+}
+
+// Find every cycle reachable in the dependency subgraph (DFS with a recursion stack).
+function findCycles(nodeIds, adj) {
+  const WHITE = 0, GRAY = 1, BLACK = 2;
+  const color = new Map(nodeIds.map(id => [id, WHITE]));
+  const stack = [];
+  const cycles = [];
+  const seenCycle = new Set();
+
+  function dfs(u) {
+    color.set(u, GRAY);
+    stack.push(u);
+    for (const v of adj.get(u) || []) {
+      if (!color.has(v)) continue;            // edge to a node outside the set (unresolved)
+      if (color.get(v) === GRAY) {
+        // back-edge → cycle from v..u then back to v
+        const i = stack.indexOf(v);
+        const cycle = stack.slice(i).concat(v);
+        const key = [...cycle].sort().join('|');
+        if (!seenCycle.has(key)) { seenCycle.add(key); cycles.push(cycle); }
+      } else if (color.get(v) === WHITE) {
+        dfs(v);
+      }
+    }
+    stack.pop();
+    color.set(u, BLACK);
+  }
+
+  for (const id of nodeIds) if (color.get(id) === WHITE) dfs(id);
+  return cycles;
+}
+
+export function buildPlan(nameOrId) {
+  const db = getDb();
+  const res = resolveId(db, nameOrId);
+  if (res.error) return res;
+  const rootId = res.id;
+
+  // 1. Gather the transitive dependency subgraph (root + everything it reaches via callees).
+  const nodes = new Map();   // id -> { id, name, present, callees: [] }
+  const adj = new Map();     // id -> [calleeIds]
+  const unresolved = new Set();
+  const stack = [rootId];
+  const seen = new Set();
+
+  while (stack.length) {
+    const cur = stack.pop();
+    if (seen.has(cur)) continue;
+    seen.add(cur);
+    const skill = db.prepare('SELECT id, name, source FROM skills WHERE id = ?').get(cur);
+    const present = !!skill;
+    const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(cur).map(r => r.to_skill);
+    nodes.set(cur, { id: cur, name: skill?.name || cur, source: skill?.source || null, present, callees });
+    adj.set(cur, callees);
+    if (!present) { unresolved.add(cur); continue; }   // dangling ref — don't expand
+    for (const c of callees) if (!seen.has(c)) stack.push(c);
+  }
+
+  const nodeIds = [...nodes.keys()];
+
+  // 2. Cycle detection (report actual paths).
+  const cycles = findCycles(nodeIds, adj);
+  const acyclic = cycles.length === 0;
+
+  // 3. Topological order (Kahn) — dependencies (callees) before dependents.
+  //    indegree[n] = how many of n's callees are still unscheduled.
+  const inSet = new Set(nodeIds);
+  const indeg = new Map();
+  const dependents = new Map(nodeIds.map(id => [id, []]));   // callee -> [callers]
+  for (const id of nodeIds) {
+    const deps = (adj.get(id) || []).filter(c => inSet.has(c));
+    indeg.set(id, deps.length);
+    for (const d of deps) dependents.get(d).push(id);
+  }
+
+  const order = [];
+  const level = new Map();
+  let queue = nodeIds.filter(id => indeg.get(id) === 0);
+  for (const id of queue) level.set(id, 0);
+
+  while (queue.length) {
+    const next = [];
+    for (const d of queue) {
+      order.push(d);
+      for (const caller of dependents.get(d)) {
+        indeg.set(caller, indeg.get(caller) - 1);
+        level.set(caller, Math.max(level.get(caller) || 0, (level.get(d) || 0) + 1));
+        if (indeg.get(caller) === 0) next.push(caller);
+      }
+    }
+    queue = next;
+  }
+
+  // 4. Parallelizable batches — nodes at the same level have no inter-dependency.
+  const levels = [];
+  if (acyclic) {
+    const maxLevel = Math.max(0, ...[...level.values()]);
+    for (let l = 0; l <= maxLevel; l++) {
+      const batch = order.filter(id => level.get(id) === l);
+      if (batch.length) levels.push(batch);
+    }
+  }
+
+  return {
+    root: { id: rootId, name: nodes.get(rootId).name },
+    count: nodeIds.length,
+    acyclic,
+    order,                                   // dependencies first, root last
+    levels,                                  // each batch can run in parallel
+    cycles,                                  // [[a,b,c,a], …] if any
+    unresolved: [...unresolved],             // referenced but not indexed
+    nodes: Object.fromEntries(nodes),
+  };
+}