promptgraph-mcp 2.4.7 → 2.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +3 -6
  2. package/ann.js +23 -51
  3. package/api.js +200 -0
  4. package/config.js +7 -0
  5. package/db.js +20 -0
  6. package/embedder.js +10 -0
  7. package/github-import.js +102 -22
  8. package/indexer.js +45 -11
  9. package/marketplace.js +47 -5
  10. package/package.json +9 -3
  11. package/search.js +84 -20
  12. package/src/filter/hard-filter.js +2 -0
  13. package/src/filter/train.js +3 -1
  14. package/src/store/flat-store.js +61 -0
  15. package/src/store/hnsw-store.js +187 -0
  16. package/src/store/index.js +19 -0
  17. package/src/store/vector-store.js +9 -0
  18. package/validator.js +7 -1
  19. package/registry/training/bad/accepts-HISTORY.md +0 -250
  20. package/registry/training/bad/argparse-CHANGELOG.md +0 -185
  21. package/registry/training/bad/balanced-match-LICENSE.md +0 -23
  22. package/registry/training/bad/better-sqlite3-README.md +0 -99
  23. package/registry/training/bad/bindings-LICENSE.md +0 -22
  24. package/registry/training/bad/bl-LICENSE.md +0 -13
  25. package/registry/training/bad/body-parser-README.md +0 -494
  26. package/registry/training/bad/bytes-HISTORY.md +0 -97
  27. package/registry/training/bad/camelcase-README.md +0 -135
  28. package/registry/training/bad/chai-README.md +0 -162
  29. package/registry/training/bad/cli-progress-LICENSE.md +0 -24
  30. package/registry/training/bad/content-type-HISTORY.md +0 -29
  31. package/registry/training/bad/cookie-SECURITY.md +0 -25
  32. package/registry/training/bad/cookie-signature-HISTORY.md +0 -70
  33. package/registry/training/bad/cors-README.md +0 -277
  34. package/registry/training/bad/cross-spawn-README.md +0 -89
  35. package/registry/training/bad/deep-extend-CHANGELOG.md +0 -46
  36. package/registry/training/bad/depd-HISTORY.md +0 -103
  37. package/registry/training/bad/esprima-README.md +0 -46
  38. package/registry/training/bad/etag-HISTORY.md +0 -83
  39. package/registry/training/bad/expect-type-SECURITY.md +0 -14
  40. package/registry/training/bad/finalhandler-HISTORY.md +0 -239
  41. package/registry/training/bad/fresh-HISTORY.md +0 -80
  42. package/registry/training/bad/glob-LICENSE.md +0 -63
  43. package/registry/training/bad/hono-README.md +0 -85
  44. package/registry/training/bad/http-errors-HISTORY.md +0 -186
  45. package/registry/training/bad/jose-LICENSE.md +0 -21
  46. package/registry/training/bad/jose-README.md +0 -153
  47. package/registry/training/bad/js-yaml-README.md +0 -299
  48. package/registry/training/bad/json-schema-typed-LICENSE.md +0 -57
  49. package/registry/training/bad/json-stringify-safe-CHANGELOG.md +0 -14
  50. package/registry/training/bad/lru-cache-LICENSE.md +0 -55
  51. package/registry/training/bad/media-typer-HISTORY.md +0 -50
  52. package/registry/training/bad/minimatch-LICENSE.md +0 -55
  53. package/registry/training/bad/minimist-CHANGELOG.md +0 -298
  54. package/registry/training/bad/minimist-README.md +0 -121
  55. package/registry/training/bad/ms-LICENSE.md +0 -21
  56. package/registry/training/bad/negotiator-HISTORY.md +0 -114
  57. package/registry/training/bad/on-finished-HISTORY.md +0 -98
  58. package/registry/training/bad/pkce-challenge-CHANGELOG.md +0 -114
  59. package/registry/training/bad/postcss-README.md +0 -28
  60. package/registry/training/bad/prebuild-install-CHANGELOG.md +0 -131
  61. package/registry/training/bad/proxy-addr-HISTORY.md +0 -161
  62. package/registry/training/bad/qs-CHANGELOG.md +0 -822
  63. package/registry/training/bad/rc-README.md +0 -227
  64. package/registry/training/bad/readable-stream-CONTRIBUTING.md +0 -38
  65. package/registry/training/bad/router-HISTORY.md +0 -228
  66. package/registry/training/bad/semver-README.md +0 -680
  67. package/registry/training/bad/send-README.md +0 -317
  68. package/registry/training/bad/serve-static-README.md +0 -253
  69. package/registry/training/bad/statuses-HISTORY.md +0 -87
  70. package/registry/training/bad/type-is-HISTORY.md +0 -292
  71. package/registry/training/bad/vary-HISTORY.md +0 -39
  72. package/registry/training/bad/vite-LICENSE.md +0 -2230
  73. package/registry/training/bad/which-CHANGELOG.md +0 -166
  74. package/registry/training/bad/zod-README.md +0 -191
  75. package/registry/training/good/skills-store-autopilot.md +0 -85
  76. package/registry/training/good/skills-store-bot-builder.md +0 -70
  77. package/registry/training/good/skills-store-chain.md +0 -136
  78. package/registry/training/good/skills-store-evolve.md +0 -100
  79. package/registry/training/good/skills-store-game.md +0 -27
  80. package/registry/training/good/skills-store-hunt.md +0 -102
  81. package/registry/training/good/skills-store-intel.md +0 -56
  82. package/registry/training/good/skills-store-memory-gc.md +0 -58
  83. package/registry/training/good/skills-store-pcsort.md +0 -207
  84. package/registry/training/good/skills-store-pickup.md +0 -60
  85. package/registry/training/good/skills-store-recon.md +0 -141
  86. package/registry/training/good/skills-store-remember.md +0 -64
  87. package/registry/training/good/skills-store-report.md +0 -117
  88. package/registry/training/good/skills-store-router.md +0 -225
  89. package/registry/training/good/skills-store-search.md +0 -168
  90. package/registry/training/good/skills-store-surface.md +0 -53
  91. package/registry/training/good/skills-store-token-scan.md +0 -141
  92. package/registry/training/good/skills-store-triage.md +0 -97
  93. package/registry/training/good/skills-store-unity.md +0 -733
  94. package/registry/training/good/skills-store-validate.md +0 -135
  95. package/registry/training/good/skills-store-web3-audit.md +0 -217
package/README.md CHANGED
@@ -154,12 +154,9 @@ Claude uses these automatically when the MCP server is running:
154
154
 
155
155
  ## Search modes
156
156
 
157
- | Mode | How | Quality |
158
- |---|---|---|
159
- | After `pg reindex` | Cosine similarity on 384-dim BGE vectors | Semantic — finds by meaning |
160
- | After `pg reindex --fast` | SQLite FTS5 BM25 | Keyword — exact word match |
161
-
162
- Search falls back to FTS5 automatically if no embeddings exist.
157
+ Search falls back to FTS5 automatically if no embeddings exist. When embeddings are
158
+ available, results are a hybrid of semantic similarity (embedding cosine) and keyword
159
+ relevance (BM25) giving you the best of both approaches.
163
160
 
164
161
  ---
165
162
 
package/ann.js CHANGED
@@ -1,61 +1,33 @@
1
1
  import { getDb, blobToVec } from './db.js';
2
+ import { getStore, resetStore } from './src/store/index.js';
2
3
 
3
- // In-memory flat index — no external dependency.
4
- // For typical skill counts (<5000) this is faster than vectra's disk-based HNSW
5
- // because all data fits in RAM and no I/O is needed per query.
6
- let _cache = null;
7
- let _cacheChunkCount = -1;
8
-
9
- function loadCache(db) {
10
- const count = db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
11
- if (_cache && count === _cacheChunkCount) return _cache;
12
- const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
13
- _cache = rows.map(r => ({
14
- skill_id: r.skill_id,
15
- vec: new Float32Array(blobToVec(r.embedding)),
16
- }));
17
- _cacheChunkCount = count;
18
- return _cache;
19
- }
20
-
21
- function cosineSim(a, b) {
22
- let dot = 0, na = 0, nb = 0;
23
- for (let i = 0; i < a.length; i++) {
24
- dot += a[i] * b[i];
25
- na += a[i] * a[i];
26
- nb += b[i] * b[i];
27
- }
28
- return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-8);
29
- }
30
-
31
- // Called after reindex — invalidate cache so next search reloads
32
4
  export async function buildAnnIndex() {
33
- _cache = null;
34
- _cacheChunkCount = -1;
35
- const db = getDb();
36
- const count = db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
37
- console.error(`[PromptGraph] In-memory index ready: ${count} chunks`);
5
+ try {
6
+ const db = getDb();
7
+ const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
8
+ const entries = rows.map(r => ({
9
+ skill_id: r.skill_id,
10
+ vector: blobToVec(r.embedding),
11
+ }));
12
+ const store = getStore();
13
+ await store.build(entries);
14
+ console.error(`[PromptGraph] ANN index ready: ${store.size} vectors`);
15
+ } catch (e) {
16
+ console.error(`[PromptGraph] ANN build failed: ${e.message}`);
17
+ }
38
18
  }
39
19
 
40
20
  export async function annSearch(queryVec, topK = 20) {
41
21
  try {
42
- const db = getDb();
43
- const cache = loadCache(db);
44
- if (!cache.length) return null;
45
-
46
- const qArr = new Float32Array(queryVec);
47
- const bestBySkill = new Map();
48
- for (const entry of cache) {
49
- const score = cosineSim(qArr, entry.vec);
50
- const prev = bestBySkill.get(entry.skill_id);
51
- if (!prev || score > prev) bestBySkill.set(entry.skill_id, score);
52
- }
53
-
54
- return [...bestBySkill.entries()]
55
- .sort((a, b) => b[1] - a[1])
56
- .slice(0, topK)
57
- .map(([skill_id, score]) => ({ skill_id, score }));
58
- } catch {
22
+ const store = getStore();
23
+ if (store.size === 0) return null;
24
+ return await store.search(queryVec, topK);
25
+ } catch (e) {
26
+ console.error(`[PromptGraph] ANN search error: ${e.message}`);
59
27
  return null;
60
28
  }
61
29
  }
30
+
31
+ export function resetAnnIndex() {
32
+ resetStore();
33
+ }
package/api.js ADDED
@@ -0,0 +1,200 @@
1
+ import { search as internalSearch, getContext, listAll as internalList } from './search.js';
2
+ import { indexAll, indexSource } from './indexer.js';
3
+ import { getDb } from './db.js';
4
+ import { loadConfig } from './config.js';
5
+
6
+ /**
7
+ * Search for skills by query text.
8
+ * @param {string} query - Natural language search query
9
+ * @param {{ topK?: number, embedWeight?: number, bm25Weight?: number, source?: string, category?: string }} [options]
10
+ * @returns {Promise<Array<{id: string, name: string, description: string, source: string, score: number, snippet: string}>>}
11
+ */
12
+ export async function search(query, options = {}) {
13
+ try {
14
+ const { topK = 5, source } = options;
15
+ const results = await internalSearch(query, source ? topK * 3 : topK);
16
+ if (source) {
17
+ return results.filter(r => r.source === source).slice(0, topK);
18
+ }
19
+ return results;
20
+ } catch {
21
+ return [];
22
+ }
23
+ }
24
+
25
+ /**
26
+ * Index all .md files in a directory under a source name.
27
+ * @param {string} sourceDir - Path to directory containing .md skill files
28
+ * @param {string} sourceName - Source label stored with each skill
29
+ * @returns {Promise<{indexed: number, skipped: number, errors: number}>}
30
+ */
31
+ export async function index(sourceDir, sourceName) {
32
+ try {
33
+ if (sourceDir.includes('..')) {
34
+ return { indexed: 0, skipped: 0, errors: 1, error: 'Path traversal detected in sourceDir' };
35
+ }
36
+ const db = getDb();
37
+ const { globSync } = await import('glob');
38
+ const fs = await import('fs');
39
+ const path = await import('path');
40
+ const { createHash } = await import('crypto');
41
+ const { parseSkillFile, isSkillFile } = await import('./parser.js');
42
+ const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
43
+ const { skillId, vecToBlob } = await import('./db.js');
44
+ const { chunkText } = await import('./chunker.js');
45
+
46
+ const files = globSync(`${sourceDir}/**/*.md`);
47
+ let indexed = 0, skipped = 0, errors = 0;
48
+ const batch = [];
49
+
50
+ for (const file of files) {
51
+ try {
52
+ const stat = fs.statSync(file);
53
+ if (stat.size > 5 * 1024 * 1024) { skipped++; continue; }
54
+ const raw = fs.readFileSync(file, 'utf8');
55
+ const hash = createHash('md5').update(raw).digest('hex');
56
+ if (!isSkillFile(file, raw)) { skipped++; continue; }
57
+ const parsed = parseSkillFile(file, sourceName, { raw });
58
+ batch.push({ ...parsed, hash });
59
+ if (batch.length >= BATCH_SIZE) {
60
+ await indexBatch(db, batch);
61
+ indexed += batch.length;
62
+ batch.length = 0;
63
+ }
64
+ } catch { errors++; }
65
+ }
66
+ if (batch.length > 0) {
67
+ await indexBatch(db, batch);
68
+ indexed += batch.length;
69
+ }
70
+ return { indexed, skipped, errors };
71
+ } catch (e) {
72
+ return { indexed: 0, skipped: 0, errors: 1, error: e.message };
73
+ }
74
+ }
75
+
76
+ /**
77
+ * Remove a skill by its ID.
78
+ * @param {string} skillId - Skill ID (format: "source::name")
79
+ * @returns {{ok: boolean, error?: string}}
80
+ */
81
+ export function remove(skillId) {
82
+ try {
83
+ const db = getDb();
84
+ db.prepare('DELETE FROM skills WHERE id = ?').run(skillId);
85
+ db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(skillId);
86
+ db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(skillId, skillId);
87
+ db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(skillId);
88
+ return { ok: true };
89
+ } catch (e) {
90
+ return { ok: false, error: e.message };
91
+ }
92
+ }
93
+
94
+ /**
95
+ * Reindex changed files only (hash-based). Scans all configured sources,
96
+ * updates files whose content hash changed, and removes stale skills.
97
+ * @returns {Promise<{updated: number, removed: number, errors: number}>}
98
+ */
99
+ export async function update() {
100
+ try {
101
+ const db = getDb();
102
+ const config = loadConfig();
103
+ const { globSync } = await import('glob');
104
+ const fs = await import('fs');
105
+ const path = await import('path');
106
+ const { createHash } = await import('crypto');
107
+ const { parseSkillFile, isSkillFile } = await import('./parser.js');
108
+ const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
109
+ const { skillId, vecToBlob } = await import('./db.js');
110
+ const { chunkText } = await import('./chunker.js');
111
+ const { indexBatch } = await import('./indexer.js');
112
+
113
+ const normalizedSources = config.sources.map(s => ({ ...s, normDir: path.resolve(s.dir) }))
114
+ .sort((a, b) => b.normDir.length - a.normDir.length);
115
+
116
+ const seenFiles = new Set();
117
+ const allFiles = [];
118
+ for (const { dir, source } of normalizedSources) {
119
+ const files = globSync(`${dir}/**/*.md`);
120
+ for (const f of files) {
121
+ const norm = path.resolve(f);
122
+ if (!seenFiles.has(norm)) { seenFiles.add(norm); allFiles.push({ file: norm, source }); }
123
+ }
124
+ }
125
+
126
+ const dbByPath = new Map();
127
+ for (const row of db.prepare('SELECT id, path, hash FROM skills').all()) {
128
+ dbByPath.set(row.path, row);
129
+ }
130
+
131
+ const existingPaths = new Set(allFiles.map(f => f.file));
132
+ let removed = 0;
133
+ for (const [filePath, row] of dbByPath) {
134
+ if (!existingPaths.has(filePath)) {
135
+ db.prepare('DELETE FROM skills WHERE id = ?').run(row.id);
136
+ db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(row.id);
137
+ db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(row.id, row.id);
138
+ db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(row.id);
139
+ removed++;
140
+ }
141
+ }
142
+
143
+ let updated = 0, errors = 0, batch = [];
144
+ for (const { file, source } of allFiles) {
145
+ try {
146
+ const raw = fs.readFileSync(file, 'utf8');
147
+ const hash = createHash('md5').update(raw).digest('hex');
148
+ const dbRow = dbByPath.get(file);
149
+ if (dbRow?.hash === hash) continue;
150
+ if (!isSkillFile(file, raw)) continue;
151
+ const parsed = parseSkillFile(file, source, { raw });
152
+ batch.push({ ...parsed, hash });
153
+ if (batch.length >= BATCH_SIZE) {
154
+ await indexBatch(db, batch);
155
+ updated += batch.length;
156
+ batch.length = 0;
157
+ }
158
+ } catch { errors++; }
159
+ }
160
+ if (batch.length > 0) {
161
+ await indexBatch(db, batch);
162
+ updated += batch.length;
163
+ }
164
+
165
+ return { updated, removed, errors };
166
+ } catch (e) {
167
+ return { updated: 0, removed: 0, errors: 1, error: e.message };
168
+ }
169
+ }
170
+
171
+ /**
172
+ * Get full skill details by ID, including edges.
173
+ * @param {string} skillId
174
+ * @returns {object|null}
175
+ */
176
+ export function get(skillId) {
177
+ try {
178
+ return getContext(skillId);
179
+ } catch {
180
+ return null;
181
+ }
182
+ }
183
+
184
+ /**
185
+ * List all skills with optional filtering.
186
+ * @param {{ source?: string, category?: string, limit?: number, offset?: number }} [options]
187
+ * @returns {Array<object>}
188
+ */
189
+ export function list(options = {}) {
190
+ try {
191
+ const { source, limit, offset } = options;
192
+ let results = internalList();
193
+ if (source) results = results.filter(r => r.source === source);
194
+ if (offset) results = results.slice(offset);
195
+ if (limit) results = results.slice(0, limit);
196
+ return results;
197
+ } catch {
198
+ return [];
199
+ }
200
+ }
package/config.js CHANGED
@@ -56,3 +56,10 @@ export async function promptConfig() {
56
56
  console.log(`\nConfig saved to ${CONFIG_PATH}`);
57
57
  return config;
58
58
  }
59
+
60
+ export function sanitizePath(inputPath) {
61
+ if (inputPath.includes('..')) {
62
+ throw new Error(`Path traversal blocked: "${inputPath}"`);
63
+ }
64
+ return path.resolve(inputPath);
65
+ }
package/db.js CHANGED
@@ -63,6 +63,26 @@ export function getDb() {
63
63
  db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
64
64
  }
65
65
 
66
+ // migrate: add registry metadata columns
67
+ if (!cols.includes('version')) {
68
+ db.exec('ALTER TABLE skills ADD COLUMN version TEXT');
69
+ }
70
+ if (!cols.includes('author')) {
71
+ db.exec('ALTER TABLE skills ADD COLUMN author TEXT');
72
+ }
73
+ if (!cols.includes('license')) {
74
+ db.exec('ALTER TABLE skills ADD COLUMN license TEXT');
75
+ }
76
+ if (!cols.includes('updated_at')) {
77
+ db.exec('ALTER TABLE skills ADD COLUMN updated_at TEXT');
78
+ }
79
+ if (!cols.includes('downloads')) {
80
+ db.exec('ALTER TABLE skills ADD COLUMN downloads INTEGER DEFAULT 0');
81
+ }
82
+ if (!cols.includes('verified')) {
83
+ db.exec('ALTER TABLE skills ADD COLUMN verified INTEGER DEFAULT 0');
84
+ }
85
+
66
86
  // migrate: convert JSON text embeddings to Float32 BLOB (one-time, ~10x smaller)
67
87
  const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
68
88
  if (textEmbeddings?.n > 0) {
package/embedder.js CHANGED
@@ -4,6 +4,11 @@ import os from 'os';
4
4
 
5
5
  const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
6
6
  const BATCH_SIZE = 256;
7
+ const MAX_EMBEDDING_CALLS = 10000;
8
+ let embedCallCount = 0;
9
+
10
+ export function getEmbedCallCount() { return embedCallCount; }
11
+ export function resetEmbedCallCount() { embedCallCount = 0; }
7
12
 
8
13
  let model = null;
9
14
 
@@ -18,6 +23,7 @@ async function getModel() {
18
23
  }
19
24
 
20
25
  export async function embed(text) {
26
+ embedCallCount++;
21
27
  const m = await getModel();
22
28
  const results = [];
23
29
  for await (const batch of m.embed([text])) {
@@ -27,6 +33,10 @@ export async function embed(text) {
27
33
  }
28
34
 
29
35
  export async function embedBatch(texts) {
36
+ if (embedCallCount + texts.length > MAX_EMBEDDING_CALLS) {
37
+ throw new Error(`Embedding queue limit exceeded (max ${MAX_EMBEDDING_CALLS} chunks per session). Use --fast or reindex incrementally.`);
38
+ }
39
+ embedCallCount += texts.length;
30
40
  const m = await getModel();
31
41
  const all = [];
32
42
  for await (const batch of m.embed(texts)) {
package/github-import.js CHANGED
@@ -45,6 +45,7 @@ async function validateMdFile(file, tmpDir) {
45
45
  try {
46
46
  const content = await httpsGet(file.download_url);
47
47
  const tmpPath = path.join(tmpDir, file.name);
48
+ fs.mkdirSync(path.dirname(tmpPath), { recursive: true });
48
49
  fs.writeFileSync(tmpPath, content);
49
50
  const result = validateSkill(tmpPath);
50
51
  if (!result.ok) {
@@ -70,17 +71,28 @@ export async function validateRepoSkills(ownerRepo) {
70
71
 
71
72
  let mdFiles;
72
73
  if (detected) {
73
- // Has a skills subdirectory — list contents
74
+ // Has a skills subdirectory — use git tree API for recursive listing
74
75
  const subdir = detected.subdir;
75
- let entries;
76
76
  try {
77
- const json = await httpsGet(`https://api.github.com/repos/${ownerRepo}/contents/${subdir}`);
78
- entries = JSON.parse(json);
77
+ const treeJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}/git/trees/HEAD?recursive=1`);
78
+ const tree = JSON.parse(treeJson);
79
+ const prefix = subdir + '/';
80
+ const mdTreeEntries = (tree.tree || []).filter(f =>
81
+ f.type === 'blob' && f.path.startsWith(prefix) && f.path.endsWith('.md')
82
+ );
83
+ let branch = 'main';
84
+ try {
85
+ const repoJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}`);
86
+ branch = JSON.parse(repoJson).default_branch || 'main';
87
+ } catch {}
88
+ mdFiles = mdTreeEntries.map(f => ({
89
+ name: f.path.replace(prefix, ''),
90
+ download_url: `https://raw.githubusercontent.com/${ownerRepo}/${branch}/${f.path}`
91
+ }));
79
92
  } catch (e) {
80
93
  try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
81
94
  return { ok: false, errors: [`Failed to list ${subdir}/ contents: ${e.message}`], warnings: [] };
82
95
  }
83
- mdFiles = entries.filter(e => e.type === 'file' && e.name.endsWith('.md'));
84
96
  } else {
85
97
  // No skills subdir — fall back to root-level .md files
86
98
  try {
@@ -131,6 +143,20 @@ async function detectSkillsDirFromAPI(ownerRepo) {
131
143
  if (dirMap.has(d)) return { subdir: dirMap.get(d), label: d };
132
144
  }
133
145
 
146
+ // 1.5 Nested skills dirs (e.g. .claude/skills, .claude-plugin/commands)
147
+ for (const prefix of ['.claude', '.claude-plugin']) {
148
+ if (dirMap.has(prefix)) {
149
+ for (const d of SKILL_DIRS) {
150
+ const nested = `${prefix}/${d}`;
151
+ try {
152
+ const sub = await httpsGet(`https://api.github.com/repos/${ownerRepo}/contents/${nested}`);
153
+ const subEntries = JSON.parse(sub);
154
+ if (subEntries.length > 0) return { subdir: nested, label: nested };
155
+ } catch {}
156
+ }
157
+ }
158
+ }
159
+
134
160
  // 2. Any subdir with 2+ .md files — pick the one with most .md files
135
161
  const subdirCandidates = entries.filter(e => e.type === 'dir' && !SKIP_DIRS_API.has(e.name.toLowerCase()));
136
162
  let best = null, bestCount = 0;
@@ -152,6 +178,8 @@ const SKIP_DIRS_API = new Set([
152
178
  '.github', 'docs', 'doc', 'documentation', 'examples', 'example',
153
179
  'tests', 'test', 'assets', 'images', 'img', 'media', 'static',
154
180
  'node_modules', 'vendor', 'dist', 'build', '.git',
181
+ 'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
182
+ 'cheatsheets', 'resources',
155
183
  ]);
156
184
 
157
185
  function git(args, cwd, stdio = 'inherit') {
@@ -177,16 +205,23 @@ function sparseClone(url, dest, subdir) {
177
205
  // Try HEAD, then main, then master
178
206
  for (const branch of ['HEAD', 'main', 'master']) {
179
207
  const r = git(['checkout', branch === 'HEAD' ? 'FETCH_HEAD' : branch], dest, 'pipe');
180
- if (r.status === 0) return true;
208
+ if (r.status === 0) return finalizeCheckout(dest, true);
181
209
  }
182
210
  return false;
183
211
  }
184
212
 
213
+ // Shared skip patterns — module scope so both cleanup functions can access them
214
+ const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
215
+ const SKIP_DIRS_LOCAL = new Set([
216
+ '.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots',
217
+ 'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor',
218
+ 'dist', 'build', 'tests', 'test',
219
+ 'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
220
+ 'cheatsheets', 'resources',
221
+ ]);
222
+
185
223
  // After full-clone root: remove files that are not skills and dirs we don't need
186
224
  function cleanupRepoRoot(repoRoot) {
187
- const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
188
- const SKIP_DIRS_LOCAL = new Set(['.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots', 'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor', 'dist', 'build', 'tests', 'test']);
189
-
190
225
  let removed = 0;
191
226
  const entries = fs.readdirSync(repoRoot, { withFileTypes: true });
192
227
  for (const entry of entries) {
@@ -223,9 +258,14 @@ function cleanupRepoDir(dirPath, SKIP_RE) {
223
258
  for (const entry of entries) {
224
259
  const fullPath = path.join(dirPath, entry.name);
225
260
  if (entry.isDirectory()) {
226
- removed += cleanupRepoDir(fullPath, SKIP_RE);
227
- // Remove empty dirs after cleanup
228
- try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
261
+ // Remove entire skip dirs (e.g. references/) nested inside skills dirs
262
+ if (SKIP_DIRS_LOCAL.has(entry.name.toLowerCase())) {
263
+ fs.rmSync(fullPath, { recursive: true, force: true });
264
+ removed++;
265
+ } else {
266
+ removed += cleanupRepoDir(fullPath, SKIP_RE);
267
+ try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
268
+ }
229
269
  } else if (entry.isFile() && entry.name.endsWith('.md')) {
230
270
  const base = entry.name.replace(/\.md$/i, '').toLowerCase();
231
271
  if (SKIP_RE.test(base)) {
@@ -252,28 +292,42 @@ function removeEmptyDirs(dirPath) {
252
292
  }
253
293
  }
254
294
 
255
- // Update sparse repo fetch + reset
295
+ // After fetch/checkout: force materialization of all sparse-matched files.
296
+ // partial clone (blob:none) + checkout often skips blob download on Windows.
297
+ function forceMaterialize(dest) {
298
+ git(['checkout', 'HEAD', '--', '.'], dest, 'pipe');
299
+ }
300
+
301
+ // Update sparse repo — fetch + checkout
256
302
  function sparseUpdate(dest, subdir) {
257
303
  const fetch = git(['fetch', '--depth=1', 'origin'], dest);
258
304
  if (fetch.status !== 0) return false;
259
305
 
260
- // Ensure sparse-checkout still set correctly
261
306
  git(['sparse-checkout', 'set', '--no-cone', `${subdir}/*.md`, `${subdir}/**/*.md`], dest, 'pipe');
262
307
 
263
- for (const ref of ['origin/HEAD', 'origin/main', 'origin/master']) {
264
- const r = git(['reset', '--hard', ref], dest, 'pipe');
265
- if (r.status === 0) return true;
308
+ for (const ref of ['origin/main', 'origin/master']) {
309
+ const r = git(['checkout', ref], dest, 'pipe');
310
+ if (r.status !== 0) continue;
311
+ forceMaterialize(dest);
312
+ return true;
266
313
  }
267
314
  return false;
268
315
  }
269
316
 
317
+ // After checkout, force materialization of sparse-matched files.
318
+ function finalizeCheckout(dest, success) {
319
+ if (success) forceMaterialize(dest);
320
+ return success;
321
+ }
322
+
270
323
  // Fallback: clone root but only checkout .md files
271
324
  function fullClone(url, dest) {
272
325
  if (fs.existsSync(dest)) {
273
326
  const fetch = git(['fetch', '--depth=1', '--filter=blob:none', 'origin'], dest);
274
327
  if (fetch.status !== 0) return false;
275
328
  for (const ref of ['origin/HEAD', 'origin/main', 'origin/master']) {
276
- if (git(['reset', '--hard', ref], dest, 'pipe').status === 0) return true;
329
+ const ok = git(['reset', '--hard', ref], dest, 'pipe').status === 0;
330
+ if (ok) return finalizeCheckout(dest, true);
277
331
  }
278
332
  return false;
279
333
  }
@@ -287,13 +341,14 @@ function fullClone(url, dest) {
287
341
  if (fetch.status !== 0) return false;
288
342
  for (const branch of ['FETCH_HEAD', 'main', 'master']) {
289
343
  const r = git(['checkout', branch], dest, 'pipe');
290
- if (r.status === 0) return true;
344
+ if (r.status === 0) return finalizeCheckout(dest, true);
291
345
  }
292
346
  return false;
293
347
  }
294
348
 
295
- // After clone: detect actual skills dir on disk
349
+ // After clone: detect actual skills dir on disk (flat + nested)
296
350
  function detectSkillsDirLocal(repoRoot) {
351
+ // 1. Flat dirs matching SKILL_DIRS (e.g. skills/, commands/)
297
352
  for (const dir of SKILL_DIRS) {
298
353
  const candidate = path.join(repoRoot, dir);
299
354
  if (fs.existsSync(candidate)) {
@@ -301,6 +356,16 @@ function detectSkillsDirLocal(repoRoot) {
301
356
  if (files.length >= 1) return { dir: candidate, label: dir, sparse: true };
302
357
  }
303
358
  }
359
+ // 2. Nested dirs: .claude/skills, .claude-plugin/commands, etc.
360
+ for (const prefix of ['.claude', '.claude-plugin']) {
361
+ for (const dir of SKILL_DIRS) {
362
+ const candidate = path.join(repoRoot, prefix, dir);
363
+ if (fs.existsSync(candidate)) {
364
+ const files = globSync(`${candidate}/**/*.md`);
365
+ if (files.length >= 1) return { dir: candidate, label: `${prefix}/${dir}`, sparse: true };
366
+ }
367
+ }
368
+ }
304
369
  return { dir: repoRoot, label: '(root)', sparse: false };
305
370
  }
306
371
 
@@ -356,7 +421,17 @@ export async function importFromGitHub(repoUrl) {
356
421
  const sparseList = isSparse
357
422
  ? spawnSync('git', ['sparse-checkout', 'list'], { cwd: dest, encoding: 'utf8' }).stdout.trim()
358
423
  : '';
359
- skillsSubdir = sparseList.split('\n').find(l => SKILL_DIRS.includes(l.trim())) || null;
424
+ skillsSubdir = (() => {
425
+ const lines = sparseList.split('\n').map(l => l.trim()).filter(Boolean);
426
+ if (lines.length === 0) return null;
427
+ const exact = lines.find(l => SKILL_DIRS.includes(l));
428
+ if (exact) return exact;
429
+ for (const line of lines) {
430
+ const m = line.match(/^(.+)\/(?:\*\*\/)?\*\.md$/);
431
+ if (m && !m[1].includes('*')) return m[1];
432
+ }
433
+ return null;
434
+ })();
360
435
 
361
436
  cloneOk = skillsSubdir
362
437
  ? sparseUpdate(dest, skillsSubdir)
@@ -381,7 +456,12 @@ export async function importFromGitHub(repoUrl) {
381
456
  removeEmptyDirs(dest);
382
457
  }
383
458
 
384
- const { dir: skillsDir, label } = detectSkillsDirLocal(dest);
459
+ const { dir: localDir, label: localLabel } = detectSkillsDirLocal(dest);
460
+ // Prefer the known skillsSubdir (from API detection or sparse patterns) as the
461
+ // canonical skills directory — it's more accurate than local heuristics for
462
+ // repos with non-standard dir names (e.g. "specialized", "cli", or nested paths)
463
+ const skillsDir = skillsSubdir ? path.join(dest, skillsSubdir) : localDir;
464
+ const label = skillsSubdir || localLabel;
385
465
  const mdFiles = globSync(`${skillsDir}/**/*.md`);
386
466
 
387
467
  if (skillsSubdir) {