promptgraph-mcp 2.9.56 → 2.9.58

package/README.md

@@ -11,10 +11,11 @@ Instead of loading every `.md` skill into context, Claude calls `pg_search` and
 
 ## Features
 
-- **Semantic search** — BGE-Small-EN embeddings, local, no API key required
+- **Semantic search** — BGE-Small-EN embeddings (fastembed), local, no API key required
 - **Token savings** — loads one skill on demand instead of all `.md` files (~20k+ tokens/session)
 - **Fast reindex** — persistent embed cache makes re-indexing unchanged skills near-instant
-- **Low RAM** — first index uses ~0.85 GB (vs 5+ GB in older versions)
+- **Flexible layouts** — finds skills in nested subfolders and in `.github/`, `.claude/`, `.opencode/` dot-folders (GitHub Copilot / agent conventions)
+- **Any local folder** — `pg add-dir <path>` indexes a folder that isn't a default source
 - **Marketplace** — browse and install community skill bundles via TUI or MCP tools
 - **Publishing** — publish skills/bundles to the registry hands-off via GitHub CLI
 - **Multi-platform** — Claude Code, OpenCode, Claude Desktop, Cursor, Windsurf, Cline, Codex
@@ -87,6 +88,44 @@ pg bundle add-repo <owner/repo> # Publish your GitHub repo to the registry (requ
 
 ---
 
+## Where your skills can live
+
+PromptGraph scans each registered source folder **recursively**, so any layout works:
+
+```
+my-skills/
+  commit-message.md                  # flat file — name from filename
+  refactor/SKILL.md                  # folder skill — name from the folder ("refactor")
+  cloud/deploy/SKILL.md              # nested categories — all found
+  .github/skills/my-skill/SKILL.md   # GitHub Copilot / agent convention
+  .claude/skills/...                 # Claude dot-folder
+```
+
+- **Subfolders** at any depth are scanned.
+- **Generic filenames** (`SKILL.md`, `index.md`, `agent.md`, …) take their name from the parent folder, so `cloud/deploy/SKILL.md` and `cloud/infra/SKILL.md` stay distinct instead of colliding.
+- **Dot-folders** `.github/{skills,prompts,agents,commands}`, `.claude/`, `.opencode/` are indexed. Other `.github` paths (`workflows`, `plugins`, …) are skipped.
+- **Doc/meta files** (`README`, `CHANGELOG`, `LICENSE`, files in `docs/`, `tests/`, `assets/`, …) are filtered out automatically.
+
+To index a folder that isn't a configured source:
+
+```bash
+pg add-dir /path/to/skills      # registers it + indexes recursively
+pg status                       # confirm it shows up with the right count
+```
+
+---
+
+## Data & config location
+
+PromptGraph stores its config, SQLite DB, and ANN index in:
+
+- `~/.promptgraph/` — fresh installs on any platform
+- `~/.claude/.promptgraph/` — kept for existing Claude Code installs (not migrated, so the index isn't orphaned)
+
+Your **skill files** stay wherever you put them (the platform skills dir or any `add-dir` folder) — only the derived index lives here.
+
+---
+
 ## OpenCode — `/pg` slash commands
 
 After `pg setup opencode`, two slash commands are available inside OpenCode:
@@ -135,12 +174,12 @@ The registry bot reads the submitted JSON and publishes within minutes.
 
 ## Skill bundles with tools
 
-A bundle can ship both skill files (`.md`) and tool scripts (`.py`, `.sh`, `.js`, etc.). When installed:
+A bundle can ship both skill files (`.md`) and tool scripts (`.py`, `.sh`, `.bash`, `.js`, `.ts`, `.rb`). When installed:
 - All `.md` files go to the skills directory
-- Script files are placed alongside them
+- Script files are cloned alongside them (sparse-checkout fetches them with the skills)
 - On Linux/macOS scripts are made executable (`chmod +x`)
 
-Bundle authors: set `has_tools: true` and include `tool_files` entries in your bundle manifest.
+For **repo bundles** (`pg bundle add-repo <owner/repo>`), scripts are detected automatically — `has_tools` and the 🔧 badge are set from the real files in the repo, no manifest needed. For **skill-list bundles**, list scripts explicitly in `tool_files`.
 
 ---
 
@@ -157,6 +196,12 @@ Bundle authors: set `has_tools: true` and include `tool_files` entries in your b
 
 ## Troubleshooting
 
+**Only some of my skills got indexed:**
+Check `pg status` for the `Found N files` count. If it's lower than expected:
+- The skills live outside a registered source → `pg add-dir <that-folder>`.
+- Generic `SKILL.md` files used to collide on name (fixed) — make sure you're on the latest version (`pg update`).
+- Files named like docs (`README`, `01-intro`, …) or under `docs/`/`tests/` are filtered by design.
+
 **Marketplace shows no bundles:**
 ```bash
 pg doctor --reset-dead

package/api.js

@@ -44,7 +44,7 @@ export async function index(sourceDir, sourceName) {
     const { skillId, vecToBlob } = await import('./db.js');
     const { chunkText } = await import('./chunker.js');
 
-    const files = globSync(`${sourceDir}/**/*.md`);
+    const files = globSync(`${sourceDir}/**/*.md`, { dot: true });
     let indexed = 0, skipped = 0, errors = 0;
     const batch = [];
 
@@ -118,7 +118,7 @@ export async function update() {
     const seenFiles = new Set();
     const allFiles = [];
     for (const { dir, source } of normalizedSources) {
-      const files = globSync(`${dir}/**/*.md`);
+      const files = globSync(`${dir}/**/*.md`, { dot: true });
       for (const f of files) {
         const norm = path.resolve(f);
         if (!seenFiles.has(norm)) { seenFiles.add(norm); allFiles.push({ file: norm, source }); }

package/commands/add-dir.js

@@ -21,7 +21,7 @@ export default async function handler(args, bin) {
   }
 
   const { globSync } = await import('glob');
-  const mdCount = globSync(`${abs}/**/*.md`, { absolute: true }).length;
+  const mdCount = globSync(`${abs}/**/*.md`, { absolute: true, dot: true }).length;
   if (mdCount === 0) {
     error(`No .md files found under ${abs}`);
     process.exit(1);

package/commands/marketplace.js

@@ -1,157 +1,157 @@
-import { colors, banner, success, error, info, section, table } from '../cli.js';
-import chalk from 'chalk';
-import path from 'path';
-import os from 'os';
-import fs from 'fs';
-
-export default async function handler(args, bin) {
-  // Subcommand: validate / prune all installed marketplace files
-  if (args[0] === 'validate' || args[0] === 'prune' || args[0] === '--validate' || args[0] === '--prune') {
-    const { validateAndPruneMarketplace } = await import('../marketplace.js');
-    const result = validateAndPruneMarketplace();
-    if (result.removed.length > 0) {
-      error(`Removed ${result.removed.length} invalid files:`);
-      result.removed.forEach(r => console.log(`  ${chalk.red('✗')} ${r.file}`));
-    }
-    if (result.errors.length > 0) {
-      error(`${result.errors.length} errors:`);
-      result.errors.forEach(e => console.log(`  ${chalk.yellow('⚠')} ${e}`));
-    }
-    success(`${result.valid.length} valid files, ${result.removed.length} removed, ${result.errors.length} errors`);
-    process.exit(result.errors.length > 0 ? 1 : 0);
-  }
-
-  if (!process.stdout.isTTY) {
-    error('marketplace TUI requires an interactive terminal');
-    process.exit(1);
-  }
-  const { browseMarketplace, browseBundles, installSkill, installBundle, installBundleBg, validateAndPruneMarketplace } = await import('../marketplace.js');
-  const { loadConfig: _lcMkt } = await import('../config.js');
-  const { getDb: _getDbMkt } = await import('../db.js');
-  const { spinner: spin2 } = await import('../cli.js');
-  const sp = spin2('Fetching marketplace...');
-  sp.start();
-  try {
-    var [skills, bundles] = await Promise.all([browseMarketplace(1000), browseBundles(1000)]);
-  } finally {
-    sp.stop();
-  }
-
-  if (skills?.error) { error(skills.error); process.exit(1); }
-
-  const { getCachedCount, setCachedCount, refreshCountsInBackground } = await import('../bundle-counts.js');
-  const { SKILLS_STORE_DIR } = await import('../config.js');
-  const { globSync } = await import('glob');
-  const { SCRIPT_GLOBS } = await import('../github-import.js');
-  const githubDir = path.join(SKILLS_STORE_DIR, 'github');
-
-  // For each bundle: if installed on disk — use real file count + detect scripts; otherwise use cache
-  const bundlesWithCounts = (Array.isArray(bundles) ? bundles : []).map(b => {
-    if (!b.repo_url) return b;
-    const owner = b.repo_url.split('/')[0];
-    const repo  = b.repo_url.split('/')[1];
-    const clonedDir = path.join(githubDir, `${owner}-${repo}`);
-    if (fs.existsSync(clonedDir) && fs.readdirSync(clonedDir).length > 0) {
-      const realCount = globSync(`${clonedDir}/**/*.md`, { absolute: true }).length;
-      const hasScripts = globSync(SCRIPT_GLOBS.map(p => `${clonedDir}/${p}`), { absolute: true }).length > 0;
-      setCachedCount(b.repo_url, realCount);
-      return { ...b, skillCount: realCount, has_tools: b.has_tools || hasScripts };
-    }
-    const cached = getCachedCount(b.repo_url);
-    const knownCount = cached ?? b.skill_count ?? null;
-    return knownCount !== null ? { ...b, skillCount: knownCount, has_tools: b.has_tools } : b;
-  });
-  refreshCountsInBackground(bundlesWithCounts);
-
-  const installedSet = new Set();
-  try {
-    const cfg = _lcMkt();
-    const db = _getDbMkt();
-
-    for (const b of (Array.isArray(bundles) ? bundles : [])) {
-      if (b.repo_url) {
-        const owner = b.repo_url.split('/')[0];
-        const repo  = b.repo_url.split('/')[1];
-        const clonedDir = path.join(githubDir, `${owner}-${repo}`);
-        if (fs.existsSync(clonedDir) && fs.readdirSync(clonedDir).length > 0) installedSet.add(b.id);
-      } else if (Array.isArray(b.skills)) {
-        const allOnDisk = b.skills.every(sid => {
-          const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(sid);
-          return row && fs.existsSync(row.path);
-        });
-        if (b.skills.length > 0 && allOnDisk) installedSet.add(b.id);
-      }
-    }
-
-    for (const row of db.prepare('SELECT id, path FROM skills WHERE source = ?').all('marketplace')) {
-      if (fs.existsSync(row.path)) installedSet.add(row.id);
-    }
-  } catch {}
-
-  const { runTUI } = await import('../tui.js');
-  const { loadConfig: _lcR, saveConfig: _scR, SKILLS_STORE_DIR: _ssR } = await import('../config.js');
-  const { getDb: _getDbR } = await import('../db.js');
-
-  await runTUI(
-    Array.isArray(skills) ? skills : [],
-    bundlesWithCounts,
-    async (item, onStatus) => {
-      if (item.type === 'bundle') {
-        const r = await installBundleBg(item.id, async (err, result) => {
-          if (err) { onStatus(false, err.message?.slice(0, 60) || 'Install failed'); return; }
-          installedSet.add(item.id);
-          const { getCachedCount } = await import('../bundle-counts.js');
-          const cached = getCachedCount(item.repo_url);
-          if (cached !== null) item.skillCount = cached;
-          validateAndPruneMarketplace();
-          onStatus(true, `Installed ${item.name}`);
-        });
-        if (r?.error) {
-          if (!r.dedup) onStatus(false, r.error.slice(0, 60));
-          return;
-        }
-        onStatus(null, `Queued ${item.name}…`);
-      } else {
-        const r = await installSkill(item.code || item.id);
-        if (r?.error) { onStatus(false, r.error.slice(0, 60)); return; }
-        installedSet.add(item.id);
-        if (item.code) installedSet.add(item.code);
-        validateAndPruneMarketplace();
-        onStatus(true, `Installed ${item.name}`);
-      }
-    },
-    installedSet,
-    async (item) => {
-      const cfg = _lcR();
-      const db  = _getDbR();
-      if (item.type === 'bundle' && item.repo_url) {
-        const owner = item.repo_url.split('/')[0];
-        const repo  = item.repo_url.split('/')[1];
-        const clonedName = `${owner}-${repo}`;
-        const clonedDir  = path.join(_ssR, 'github', clonedName);
-        if (fs.existsSync(clonedDir)) fs.rmSync(clonedDir, { recursive: true, force: true });
-        const src = `github:${clonedName}`;
-        cfg.sources = cfg.sources.filter(s => s.source !== src && !s.dir.startsWith(clonedDir));
-        _scR(cfg);
-        db.prepare('DELETE FROM skills WHERE source = ?').run(src);
-        db.prepare('DELETE FROM chunks WHERE skill_id NOT IN (SELECT id FROM skills)').run();
-      } else if (item.type === 'bundle') {
-        const mktDir = path.join(_ssR, 'marketplace');
-        for (const sid of (item.skills || [])) {
-          const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(sid);
-          if (row?.path && fs.existsSync(row.path)) fs.unlinkSync(row.path);
-          db.prepare('DELETE FROM skills WHERE id = ?').run(sid);
-          db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(sid);
-        }
-      } else {
-        const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(item.id);
-        if (row?.path && fs.existsSync(row.path)) fs.unlinkSync(row.path);
-        db.prepare('DELETE FROM skills WHERE id = ?').run(item.id);
-        db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(item.id);
-      }
-      installedSet.delete(item.id);
-      if (item.code) installedSet.delete(item.code);
-    }
-  );
-  process.exit(0);
-}
+import { colors, banner, success, error, info, section, table } from '../cli.js';
+import chalk from 'chalk';
+import path from 'path';
+import os from 'os';
+import fs from 'fs';
+
+export default async function handler(args, bin) {
+  // Subcommand: validate / prune all installed marketplace files
+  if (args[0] === 'validate' || args[0] === 'prune' || args[0] === '--validate' || args[0] === '--prune') {
+    const { validateAndPruneMarketplace } = await import('../marketplace.js');
+    const result = validateAndPruneMarketplace();
+    if (result.removed.length > 0) {
+      error(`Removed ${result.removed.length} invalid files:`);
+      result.removed.forEach(r => console.log(`  ${chalk.red('✗')} ${r.file}`));
+    }
+    if (result.errors.length > 0) {
+      error(`${result.errors.length} errors:`);
+      result.errors.forEach(e => console.log(`  ${chalk.yellow('⚠')} ${e}`));
+    }
+    success(`${result.valid.length} valid files, ${result.removed.length} removed, ${result.errors.length} errors`);
+    process.exit(result.errors.length > 0 ? 1 : 0);
+  }
+
+  if (!process.stdout.isTTY) {
+    error('marketplace TUI requires an interactive terminal');
+    process.exit(1);
+  }
+  const { browseMarketplace, browseBundles, installSkill, installBundle, installBundleBg, validateAndPruneMarketplace } = await import('../marketplace.js');
+  const { loadConfig: _lcMkt } = await import('../config.js');
+  const { getDb: _getDbMkt } = await import('../db.js');
+  const { spinner: spin2 } = await import('../cli.js');
+  const sp = spin2('Fetching marketplace...');
+  sp.start();
+  try {
+    var [skills, bundles] = await Promise.all([browseMarketplace(1000), browseBundles(1000)]);
+  } finally {
+    sp.stop();
+  }
+
+  if (skills?.error) { error(skills.error); process.exit(1); }
+
+  const { getCachedCount, setCachedCount, refreshCountsInBackground } = await import('../bundle-counts.js');
+  const { SKILLS_STORE_DIR } = await import('../config.js');
+  const { globSync } = await import('glob');
+  const { SCRIPT_GLOBS } = await import('../github-import.js');
+  const githubDir = path.join(SKILLS_STORE_DIR, 'github');
+
+  // For each bundle: if installed on disk — use real file count + detect scripts; otherwise use cache
+  const bundlesWithCounts = (Array.isArray(bundles) ? bundles : []).map(b => {
+    if (!b.repo_url) return b;
+    const owner = b.repo_url.split('/')[0];
+    const repo  = b.repo_url.split('/')[1];
+    const clonedDir = path.join(githubDir, `${owner}-${repo}`);
+    if (fs.existsSync(clonedDir) && fs.readdirSync(clonedDir).length > 0) {
+      const realCount = globSync(`${clonedDir}/**/*.md`, { absolute: true, dot: true }).length;
+      const hasScripts = globSync(SCRIPT_GLOBS.map(p => `${clonedDir}/${p}`), { absolute: true, dot: true }).length > 0;
+      setCachedCount(b.repo_url, realCount);
+      return { ...b, skillCount: realCount, has_tools: b.has_tools || hasScripts };
+    }
+    const cached = getCachedCount(b.repo_url);
+    const knownCount = cached ?? b.skill_count ?? null;
+    return knownCount !== null ? { ...b, skillCount: knownCount, has_tools: b.has_tools } : b;
+  });
+  refreshCountsInBackground(bundlesWithCounts);
+
+  const installedSet = new Set();
+  try {
+    const cfg = _lcMkt();
+    const db = _getDbMkt();
+
+    for (const b of (Array.isArray(bundles) ? bundles : [])) {
+      if (b.repo_url) {
+        const owner = b.repo_url.split('/')[0];
+        const repo  = b.repo_url.split('/')[1];
+        const clonedDir = path.join(githubDir, `${owner}-${repo}`);
+        if (fs.existsSync(clonedDir) && fs.readdirSync(clonedDir).length > 0) installedSet.add(b.id);
+      } else if (Array.isArray(b.skills)) {
+        const allOnDisk = b.skills.every(sid => {
+          const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(sid);
+          return row && fs.existsSync(row.path);
+        });
+        if (b.skills.length > 0 && allOnDisk) installedSet.add(b.id);
+      }
+    }
+
+    for (const row of db.prepare('SELECT id, path FROM skills WHERE source = ?').all('marketplace')) {
+      if (fs.existsSync(row.path)) installedSet.add(row.id);
+    }
+  } catch {}
+
+  const { runTUI } = await import('../tui.js');
+  const { loadConfig: _lcR, saveConfig: _scR, SKILLS_STORE_DIR: _ssR } = await import('../config.js');
+  const { getDb: _getDbR } = await import('../db.js');
+
+  await runTUI(
+    Array.isArray(skills) ? skills : [],
+    bundlesWithCounts,
+    async (item, onStatus) => {
+      if (item.type === 'bundle') {
+        const r = await installBundleBg(item.id, async (err, result) => {
+          if (err) { onStatus(false, err.message?.slice(0, 60) || 'Install failed'); return; }
+          installedSet.add(item.id);
+          const { getCachedCount } = await import('../bundle-counts.js');
+          const cached = getCachedCount(item.repo_url);
+          if (cached !== null) item.skillCount = cached;
+          validateAndPruneMarketplace();
+          onStatus(true, `Installed ${item.name}`);
+        });
+        if (r?.error) {
+          if (!r.dedup) onStatus(false, r.error.slice(0, 60));
+          return;
+        }
+        onStatus(null, `Queued ${item.name}…`);
+      } else {
+        const r = await installSkill(item.code || item.id);
+        if (r?.error) { onStatus(false, r.error.slice(0, 60)); return; }
+        installedSet.add(item.id);
+        if (item.code) installedSet.add(item.code);
+        validateAndPruneMarketplace();
+        onStatus(true, `Installed ${item.name}`);
+      }
+    },
+    installedSet,
+    async (item) => {
+      const cfg = _lcR();
+      const db  = _getDbR();
+      if (item.type === 'bundle' && item.repo_url) {
+        const owner = item.repo_url.split('/')[0];
+        const repo  = item.repo_url.split('/')[1];
+        const clonedName = `${owner}-${repo}`;
+        const clonedDir  = path.join(_ssR, 'github', clonedName);
+        if (fs.existsSync(clonedDir)) fs.rmSync(clonedDir, { recursive: true, force: true });
+        const src = `github:${clonedName}`;
+        cfg.sources = cfg.sources.filter(s => s.source !== src && !s.dir.startsWith(clonedDir));
+        _scR(cfg);
+        db.prepare('DELETE FROM skills WHERE source = ?').run(src);
+        db.prepare('DELETE FROM chunks WHERE skill_id NOT IN (SELECT id FROM skills)').run();
+      } else if (item.type === 'bundle') {
+        const mktDir = path.join(_ssR, 'marketplace');
+        for (const sid of (item.skills || [])) {
+          const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(sid);
+          if (row?.path && fs.existsSync(row.path)) fs.unlinkSync(row.path);
+          db.prepare('DELETE FROM skills WHERE id = ?').run(sid);
+          db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(sid);
+        }
+      } else {
+        const row = db.prepare('SELECT path FROM skills WHERE id = ?').get(item.id);
+        if (row?.path && fs.existsSync(row.path)) fs.unlinkSync(row.path);
+        db.prepare('DELETE FROM skills WHERE id = ?').run(item.id);
+        db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(item.id);
+      }
+      installedSet.delete(item.id);
+      if (item.code) installedSet.delete(item.code);
+    }
+  );
+  process.exit(0);
+}