promptgraph-mcp 2.4.7 → 2.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -6
- package/ann.js +23 -51
- package/api.js +200 -0
- package/config.js +7 -0
- package/db.js +20 -0
- package/embedder.js +10 -0
- package/github-import.js +102 -22
- package/indexer.js +45 -11
- package/marketplace.js +47 -5
- package/package.json +9 -3
- package/search.js +84 -20
- package/src/filter/hard-filter.js +2 -0
- package/src/filter/train.js +3 -1
- package/src/store/flat-store.js +61 -0
- package/src/store/hnsw-store.js +187 -0
- package/src/store/index.js +19 -0
- package/src/store/vector-store.js +9 -0
- package/validator.js +7 -1
- package/registry/training/bad/accepts-HISTORY.md +0 -250
- package/registry/training/bad/argparse-CHANGELOG.md +0 -185
- package/registry/training/bad/balanced-match-LICENSE.md +0 -23
- package/registry/training/bad/better-sqlite3-README.md +0 -99
- package/registry/training/bad/bindings-LICENSE.md +0 -22
- package/registry/training/bad/bl-LICENSE.md +0 -13
- package/registry/training/bad/body-parser-README.md +0 -494
- package/registry/training/bad/bytes-HISTORY.md +0 -97
- package/registry/training/bad/camelcase-README.md +0 -135
- package/registry/training/bad/chai-README.md +0 -162
- package/registry/training/bad/cli-progress-LICENSE.md +0 -24
- package/registry/training/bad/content-type-HISTORY.md +0 -29
- package/registry/training/bad/cookie-SECURITY.md +0 -25
- package/registry/training/bad/cookie-signature-HISTORY.md +0 -70
- package/registry/training/bad/cors-README.md +0 -277
- package/registry/training/bad/cross-spawn-README.md +0 -89
- package/registry/training/bad/deep-extend-CHANGELOG.md +0 -46
- package/registry/training/bad/depd-HISTORY.md +0 -103
- package/registry/training/bad/esprima-README.md +0 -46
- package/registry/training/bad/etag-HISTORY.md +0 -83
- package/registry/training/bad/expect-type-SECURITY.md +0 -14
- package/registry/training/bad/finalhandler-HISTORY.md +0 -239
- package/registry/training/bad/fresh-HISTORY.md +0 -80
- package/registry/training/bad/glob-LICENSE.md +0 -63
- package/registry/training/bad/hono-README.md +0 -85
- package/registry/training/bad/http-errors-HISTORY.md +0 -186
- package/registry/training/bad/jose-LICENSE.md +0 -21
- package/registry/training/bad/jose-README.md +0 -153
- package/registry/training/bad/js-yaml-README.md +0 -299
- package/registry/training/bad/json-schema-typed-LICENSE.md +0 -57
- package/registry/training/bad/json-stringify-safe-CHANGELOG.md +0 -14
- package/registry/training/bad/lru-cache-LICENSE.md +0 -55
- package/registry/training/bad/media-typer-HISTORY.md +0 -50
- package/registry/training/bad/minimatch-LICENSE.md +0 -55
- package/registry/training/bad/minimist-CHANGELOG.md +0 -298
- package/registry/training/bad/minimist-README.md +0 -121
- package/registry/training/bad/ms-LICENSE.md +0 -21
- package/registry/training/bad/negotiator-HISTORY.md +0 -114
- package/registry/training/bad/on-finished-HISTORY.md +0 -98
- package/registry/training/bad/pkce-challenge-CHANGELOG.md +0 -114
- package/registry/training/bad/postcss-README.md +0 -28
- package/registry/training/bad/prebuild-install-CHANGELOG.md +0 -131
- package/registry/training/bad/proxy-addr-HISTORY.md +0 -161
- package/registry/training/bad/qs-CHANGELOG.md +0 -822
- package/registry/training/bad/rc-README.md +0 -227
- package/registry/training/bad/readable-stream-CONTRIBUTING.md +0 -38
- package/registry/training/bad/router-HISTORY.md +0 -228
- package/registry/training/bad/semver-README.md +0 -680
- package/registry/training/bad/send-README.md +0 -317
- package/registry/training/bad/serve-static-README.md +0 -253
- package/registry/training/bad/statuses-HISTORY.md +0 -87
- package/registry/training/bad/type-is-HISTORY.md +0 -292
- package/registry/training/bad/vary-HISTORY.md +0 -39
- package/registry/training/bad/vite-LICENSE.md +0 -2230
- package/registry/training/bad/which-CHANGELOG.md +0 -166
- package/registry/training/bad/zod-README.md +0 -191
- package/registry/training/good/skills-store-autopilot.md +0 -85
- package/registry/training/good/skills-store-bot-builder.md +0 -70
- package/registry/training/good/skills-store-chain.md +0 -136
- package/registry/training/good/skills-store-evolve.md +0 -100
- package/registry/training/good/skills-store-game.md +0 -27
- package/registry/training/good/skills-store-hunt.md +0 -102
- package/registry/training/good/skills-store-intel.md +0 -56
- package/registry/training/good/skills-store-memory-gc.md +0 -58
- package/registry/training/good/skills-store-pcsort.md +0 -207
- package/registry/training/good/skills-store-pickup.md +0 -60
- package/registry/training/good/skills-store-recon.md +0 -141
- package/registry/training/good/skills-store-remember.md +0 -64
- package/registry/training/good/skills-store-report.md +0 -117
- package/registry/training/good/skills-store-router.md +0 -225
- package/registry/training/good/skills-store-search.md +0 -168
- package/registry/training/good/skills-store-surface.md +0 -53
- package/registry/training/good/skills-store-token-scan.md +0 -141
- package/registry/training/good/skills-store-triage.md +0 -97
- package/registry/training/good/skills-store-unity.md +0 -733
- package/registry/training/good/skills-store-validate.md +0 -135
- package/registry/training/good/skills-store-web3-audit.md +0 -217
package/README.md
CHANGED
|
@@ -154,12 +154,9 @@ Claude uses these automatically when the MCP server is running:
|
|
|
154
154
|
|
|
155
155
|
## Search modes
|
|
156
156
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
| After `pg reindex --fast` | SQLite FTS5 BM25 | Keyword — exact word match |
|
|
161
|
-
|
|
162
|
-
Search falls back to FTS5 automatically if no embeddings exist.
|
|
157
|
+
Search falls back to FTS5 automatically if no embeddings exist. When embeddings are
|
|
158
|
+
available, results are a hybrid of semantic similarity (embedding cosine) and keyword
|
|
159
|
+
relevance (BM25) — giving you the best of both approaches.
|
|
163
160
|
|
|
164
161
|
---
|
|
165
162
|
|
package/ann.js
CHANGED
|
@@ -1,61 +1,33 @@
|
|
|
1
1
|
import { getDb, blobToVec } from './db.js';
|
|
2
|
+
import { getStore, resetStore } from './src/store/index.js';
|
|
2
3
|
|
|
3
|
-
// In-memory flat index — no external dependency.
|
|
4
|
-
// For typical skill counts (<5000) this is faster than vectra's disk-based HNSW
|
|
5
|
-
// because all data fits in RAM and no I/O is needed per query.
|
|
6
|
-
let _cache = null;
|
|
7
|
-
let _cacheChunkCount = -1;
|
|
8
|
-
|
|
9
|
-
function loadCache(db) {
|
|
10
|
-
const count = db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
|
|
11
|
-
if (_cache && count === _cacheChunkCount) return _cache;
|
|
12
|
-
const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
|
|
13
|
-
_cache = rows.map(r => ({
|
|
14
|
-
skill_id: r.skill_id,
|
|
15
|
-
vec: new Float32Array(blobToVec(r.embedding)),
|
|
16
|
-
}));
|
|
17
|
-
_cacheChunkCount = count;
|
|
18
|
-
return _cache;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
function cosineSim(a, b) {
|
|
22
|
-
let dot = 0, na = 0, nb = 0;
|
|
23
|
-
for (let i = 0; i < a.length; i++) {
|
|
24
|
-
dot += a[i] * b[i];
|
|
25
|
-
na += a[i] * a[i];
|
|
26
|
-
nb += b[i] * b[i];
|
|
27
|
-
}
|
|
28
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-8);
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
// Called after reindex — invalidate cache so next search reloads
|
|
32
4
|
export async function buildAnnIndex() {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
5
|
+
try {
|
|
6
|
+
const db = getDb();
|
|
7
|
+
const rows = db.prepare('SELECT skill_id, embedding FROM chunks').all();
|
|
8
|
+
const entries = rows.map(r => ({
|
|
9
|
+
skill_id: r.skill_id,
|
|
10
|
+
vector: blobToVec(r.embedding),
|
|
11
|
+
}));
|
|
12
|
+
const store = getStore();
|
|
13
|
+
await store.build(entries);
|
|
14
|
+
console.error(`[PromptGraph] ANN index ready: ${store.size} vectors`);
|
|
15
|
+
} catch (e) {
|
|
16
|
+
console.error(`[PromptGraph] ANN build failed: ${e.message}`);
|
|
17
|
+
}
|
|
38
18
|
}
|
|
39
19
|
|
|
40
20
|
export async function annSearch(queryVec, topK = 20) {
|
|
41
21
|
try {
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const bestBySkill = new Map();
|
|
48
|
-
for (const entry of cache) {
|
|
49
|
-
const score = cosineSim(qArr, entry.vec);
|
|
50
|
-
const prev = bestBySkill.get(entry.skill_id);
|
|
51
|
-
if (!prev || score > prev) bestBySkill.set(entry.skill_id, score);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
return [...bestBySkill.entries()]
|
|
55
|
-
.sort((a, b) => b[1] - a[1])
|
|
56
|
-
.slice(0, topK)
|
|
57
|
-
.map(([skill_id, score]) => ({ skill_id, score }));
|
|
58
|
-
} catch {
|
|
22
|
+
const store = getStore();
|
|
23
|
+
if (store.size === 0) return null;
|
|
24
|
+
return await store.search(queryVec, topK);
|
|
25
|
+
} catch (e) {
|
|
26
|
+
console.error(`[PromptGraph] ANN search error: ${e.message}`);
|
|
59
27
|
return null;
|
|
60
28
|
}
|
|
61
29
|
}
|
|
30
|
+
|
|
31
|
+
export function resetAnnIndex() {
|
|
32
|
+
resetStore();
|
|
33
|
+
}
|
package/api.js
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { search as internalSearch, getContext, listAll as internalList } from './search.js';
|
|
2
|
+
import { indexAll, indexSource } from './indexer.js';
|
|
3
|
+
import { getDb } from './db.js';
|
|
4
|
+
import { loadConfig } from './config.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Search for skills by query text.
|
|
8
|
+
* @param {string} query - Natural language search query
|
|
9
|
+
* @param {{ topK?: number, embedWeight?: number, bm25Weight?: number, source?: string, category?: string }} [options]
|
|
10
|
+
* @returns {Promise<Array<{id: string, name: string, description: string, source: string, score: number, snippet: string}>>}
|
|
11
|
+
*/
|
|
12
|
+
export async function search(query, options = {}) {
|
|
13
|
+
try {
|
|
14
|
+
const { topK = 5, source } = options;
|
|
15
|
+
const results = await internalSearch(query, source ? topK * 3 : topK);
|
|
16
|
+
if (source) {
|
|
17
|
+
return results.filter(r => r.source === source).slice(0, topK);
|
|
18
|
+
}
|
|
19
|
+
return results;
|
|
20
|
+
} catch {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Index all .md files in a directory under a source name.
|
|
27
|
+
* @param {string} sourceDir - Path to directory containing .md skill files
|
|
28
|
+
* @param {string} sourceName - Source label stored with each skill
|
|
29
|
+
* @returns {Promise<{indexed: number, skipped: number, errors: number}>}
|
|
30
|
+
*/
|
|
31
|
+
export async function index(sourceDir, sourceName) {
|
|
32
|
+
try {
|
|
33
|
+
if (sourceDir.includes('..')) {
|
|
34
|
+
return { indexed: 0, skipped: 0, errors: 1, error: 'Path traversal detected in sourceDir' };
|
|
35
|
+
}
|
|
36
|
+
const db = getDb();
|
|
37
|
+
const { globSync } = await import('glob');
|
|
38
|
+
const fs = await import('fs');
|
|
39
|
+
const path = await import('path');
|
|
40
|
+
const { createHash } = await import('crypto');
|
|
41
|
+
const { parseSkillFile, isSkillFile } = await import('./parser.js');
|
|
42
|
+
const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
|
|
43
|
+
const { skillId, vecToBlob } = await import('./db.js');
|
|
44
|
+
const { chunkText } = await import('./chunker.js');
|
|
45
|
+
|
|
46
|
+
const files = globSync(`${sourceDir}/**/*.md`);
|
|
47
|
+
let indexed = 0, skipped = 0, errors = 0;
|
|
48
|
+
const batch = [];
|
|
49
|
+
|
|
50
|
+
for (const file of files) {
|
|
51
|
+
try {
|
|
52
|
+
const stat = fs.statSync(file);
|
|
53
|
+
if (stat.size > 5 * 1024 * 1024) { skipped++; continue; }
|
|
54
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
55
|
+
const hash = createHash('md5').update(raw).digest('hex');
|
|
56
|
+
if (!isSkillFile(file, raw)) { skipped++; continue; }
|
|
57
|
+
const parsed = parseSkillFile(file, sourceName, { raw });
|
|
58
|
+
batch.push({ ...parsed, hash });
|
|
59
|
+
if (batch.length >= BATCH_SIZE) {
|
|
60
|
+
await indexBatch(db, batch);
|
|
61
|
+
indexed += batch.length;
|
|
62
|
+
batch.length = 0;
|
|
63
|
+
}
|
|
64
|
+
} catch { errors++; }
|
|
65
|
+
}
|
|
66
|
+
if (batch.length > 0) {
|
|
67
|
+
await indexBatch(db, batch);
|
|
68
|
+
indexed += batch.length;
|
|
69
|
+
}
|
|
70
|
+
return { indexed, skipped, errors };
|
|
71
|
+
} catch (e) {
|
|
72
|
+
return { indexed: 0, skipped: 0, errors: 1, error: e.message };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Remove a skill by its ID.
|
|
78
|
+
* @param {string} skillId - Skill ID (format: "source::name")
|
|
79
|
+
* @returns {{ok: boolean, error?: string}}
|
|
80
|
+
*/
|
|
81
|
+
export function remove(skillId) {
|
|
82
|
+
try {
|
|
83
|
+
const db = getDb();
|
|
84
|
+
db.prepare('DELETE FROM skills WHERE id = ?').run(skillId);
|
|
85
|
+
db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(skillId);
|
|
86
|
+
db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(skillId, skillId);
|
|
87
|
+
db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(skillId);
|
|
88
|
+
return { ok: true };
|
|
89
|
+
} catch (e) {
|
|
90
|
+
return { ok: false, error: e.message };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Reindex changed files only (hash-based). Scans all configured sources,
|
|
96
|
+
* updates files whose content hash changed, and removes stale skills.
|
|
97
|
+
* @returns {Promise<{updated: number, removed: number, errors: number}>}
|
|
98
|
+
*/
|
|
99
|
+
export async function update() {
|
|
100
|
+
try {
|
|
101
|
+
const db = getDb();
|
|
102
|
+
const config = loadConfig();
|
|
103
|
+
const { globSync } = await import('glob');
|
|
104
|
+
const fs = await import('fs');
|
|
105
|
+
const path = await import('path');
|
|
106
|
+
const { createHash } = await import('crypto');
|
|
107
|
+
const { parseSkillFile, isSkillFile } = await import('./parser.js');
|
|
108
|
+
const { embedBatch, BATCH_SIZE } = await import('./embedder.js');
|
|
109
|
+
const { skillId, vecToBlob } = await import('./db.js');
|
|
110
|
+
const { chunkText } = await import('./chunker.js');
|
|
111
|
+
const { indexBatch } = await import('./indexer.js');
|
|
112
|
+
|
|
113
|
+
const normalizedSources = config.sources.map(s => ({ ...s, normDir: path.resolve(s.dir) }))
|
|
114
|
+
.sort((a, b) => b.normDir.length - a.normDir.length);
|
|
115
|
+
|
|
116
|
+
const seenFiles = new Set();
|
|
117
|
+
const allFiles = [];
|
|
118
|
+
for (const { dir, source } of normalizedSources) {
|
|
119
|
+
const files = globSync(`${dir}/**/*.md`);
|
|
120
|
+
for (const f of files) {
|
|
121
|
+
const norm = path.resolve(f);
|
|
122
|
+
if (!seenFiles.has(norm)) { seenFiles.add(norm); allFiles.push({ file: norm, source }); }
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const dbByPath = new Map();
|
|
127
|
+
for (const row of db.prepare('SELECT id, path, hash FROM skills').all()) {
|
|
128
|
+
dbByPath.set(row.path, row);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const existingPaths = new Set(allFiles.map(f => f.file));
|
|
132
|
+
let removed = 0;
|
|
133
|
+
for (const [filePath, row] of dbByPath) {
|
|
134
|
+
if (!existingPaths.has(filePath)) {
|
|
135
|
+
db.prepare('DELETE FROM skills WHERE id = ?').run(row.id);
|
|
136
|
+
db.prepare('DELETE FROM chunks WHERE skill_id = ?').run(row.id);
|
|
137
|
+
db.prepare('DELETE FROM edges WHERE from_skill = ? OR to_skill = ?').run(row.id, row.id);
|
|
138
|
+
db.prepare('DELETE FROM ratings WHERE skill_id = ?').run(row.id);
|
|
139
|
+
removed++;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
let updated = 0, errors = 0, batch = [];
|
|
144
|
+
for (const { file, source } of allFiles) {
|
|
145
|
+
try {
|
|
146
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
147
|
+
const hash = createHash('md5').update(raw).digest('hex');
|
|
148
|
+
const dbRow = dbByPath.get(file);
|
|
149
|
+
if (dbRow?.hash === hash) continue;
|
|
150
|
+
if (!isSkillFile(file, raw)) continue;
|
|
151
|
+
const parsed = parseSkillFile(file, source, { raw });
|
|
152
|
+
batch.push({ ...parsed, hash });
|
|
153
|
+
if (batch.length >= BATCH_SIZE) {
|
|
154
|
+
await indexBatch(db, batch);
|
|
155
|
+
updated += batch.length;
|
|
156
|
+
batch.length = 0;
|
|
157
|
+
}
|
|
158
|
+
} catch { errors++; }
|
|
159
|
+
}
|
|
160
|
+
if (batch.length > 0) {
|
|
161
|
+
await indexBatch(db, batch);
|
|
162
|
+
updated += batch.length;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return { updated, removed, errors };
|
|
166
|
+
} catch (e) {
|
|
167
|
+
return { updated: 0, removed: 0, errors: 1, error: e.message };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Get full skill details by ID, including edges.
|
|
173
|
+
* @param {string} skillId
|
|
174
|
+
* @returns {object|null}
|
|
175
|
+
*/
|
|
176
|
+
export function get(skillId) {
|
|
177
|
+
try {
|
|
178
|
+
return getContext(skillId);
|
|
179
|
+
} catch {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* List all skills with optional filtering.
|
|
186
|
+
* @param {{ source?: string, category?: string, limit?: number, offset?: number }} [options]
|
|
187
|
+
* @returns {Array<object>}
|
|
188
|
+
*/
|
|
189
|
+
export function list(options = {}) {
|
|
190
|
+
try {
|
|
191
|
+
const { source, limit, offset } = options;
|
|
192
|
+
let results = internalList();
|
|
193
|
+
if (source) results = results.filter(r => r.source === source);
|
|
194
|
+
if (offset) results = results.slice(offset);
|
|
195
|
+
if (limit) results = results.slice(0, limit);
|
|
196
|
+
return results;
|
|
197
|
+
} catch {
|
|
198
|
+
return [];
|
|
199
|
+
}
|
|
200
|
+
}
|
package/config.js
CHANGED
|
@@ -56,3 +56,10 @@ export async function promptConfig() {
|
|
|
56
56
|
console.log(`\nConfig saved to ${CONFIG_PATH}`);
|
|
57
57
|
return config;
|
|
58
58
|
}
|
|
59
|
+
|
|
60
|
+
export function sanitizePath(inputPath) {
|
|
61
|
+
if (inputPath.includes('..')) {
|
|
62
|
+
throw new Error(`Path traversal blocked: "${inputPath}"`);
|
|
63
|
+
}
|
|
64
|
+
return path.resolve(inputPath);
|
|
65
|
+
}
|
package/db.js
CHANGED
|
@@ -63,6 +63,26 @@ export function getDb() {
|
|
|
63
63
|
db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
// migrate: add registry metadata columns
|
|
67
|
+
if (!cols.includes('version')) {
|
|
68
|
+
db.exec('ALTER TABLE skills ADD COLUMN version TEXT');
|
|
69
|
+
}
|
|
70
|
+
if (!cols.includes('author')) {
|
|
71
|
+
db.exec('ALTER TABLE skills ADD COLUMN author TEXT');
|
|
72
|
+
}
|
|
73
|
+
if (!cols.includes('license')) {
|
|
74
|
+
db.exec('ALTER TABLE skills ADD COLUMN license TEXT');
|
|
75
|
+
}
|
|
76
|
+
if (!cols.includes('updated_at')) {
|
|
77
|
+
db.exec('ALTER TABLE skills ADD COLUMN updated_at TEXT');
|
|
78
|
+
}
|
|
79
|
+
if (!cols.includes('downloads')) {
|
|
80
|
+
db.exec('ALTER TABLE skills ADD COLUMN downloads INTEGER DEFAULT 0');
|
|
81
|
+
}
|
|
82
|
+
if (!cols.includes('verified')) {
|
|
83
|
+
db.exec('ALTER TABLE skills ADD COLUMN verified INTEGER DEFAULT 0');
|
|
84
|
+
}
|
|
85
|
+
|
|
66
86
|
// migrate: convert JSON text embeddings to Float32 BLOB (one-time, ~10x smaller)
|
|
67
87
|
const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
|
|
68
88
|
if (textEmbeddings?.n > 0) {
|
package/embedder.js
CHANGED
|
@@ -4,6 +4,11 @@ import os from 'os';
|
|
|
4
4
|
|
|
5
5
|
const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
|
|
6
6
|
const BATCH_SIZE = 256;
|
|
7
|
+
const MAX_EMBEDDING_CALLS = 10000;
|
|
8
|
+
let embedCallCount = 0;
|
|
9
|
+
|
|
10
|
+
export function getEmbedCallCount() { return embedCallCount; }
|
|
11
|
+
export function resetEmbedCallCount() { embedCallCount = 0; }
|
|
7
12
|
|
|
8
13
|
let model = null;
|
|
9
14
|
|
|
@@ -18,6 +23,7 @@ async function getModel() {
|
|
|
18
23
|
}
|
|
19
24
|
|
|
20
25
|
export async function embed(text) {
|
|
26
|
+
embedCallCount++;
|
|
21
27
|
const m = await getModel();
|
|
22
28
|
const results = [];
|
|
23
29
|
for await (const batch of m.embed([text])) {
|
|
@@ -27,6 +33,10 @@ export async function embed(text) {
|
|
|
27
33
|
}
|
|
28
34
|
|
|
29
35
|
export async function embedBatch(texts) {
|
|
36
|
+
if (embedCallCount + texts.length > MAX_EMBEDDING_CALLS) {
|
|
37
|
+
throw new Error(`Embedding queue limit exceeded (max ${MAX_EMBEDDING_CALLS} chunks per session). Use --fast or reindex incrementally.`);
|
|
38
|
+
}
|
|
39
|
+
embedCallCount += texts.length;
|
|
30
40
|
const m = await getModel();
|
|
31
41
|
const all = [];
|
|
32
42
|
for await (const batch of m.embed(texts)) {
|
package/github-import.js
CHANGED
|
@@ -45,6 +45,7 @@ async function validateMdFile(file, tmpDir) {
|
|
|
45
45
|
try {
|
|
46
46
|
const content = await httpsGet(file.download_url);
|
|
47
47
|
const tmpPath = path.join(tmpDir, file.name);
|
|
48
|
+
fs.mkdirSync(path.dirname(tmpPath), { recursive: true });
|
|
48
49
|
fs.writeFileSync(tmpPath, content);
|
|
49
50
|
const result = validateSkill(tmpPath);
|
|
50
51
|
if (!result.ok) {
|
|
@@ -70,17 +71,28 @@ export async function validateRepoSkills(ownerRepo) {
|
|
|
70
71
|
|
|
71
72
|
let mdFiles;
|
|
72
73
|
if (detected) {
|
|
73
|
-
// Has a skills subdirectory —
|
|
74
|
+
// Has a skills subdirectory — use git tree API for recursive listing
|
|
74
75
|
const subdir = detected.subdir;
|
|
75
|
-
let entries;
|
|
76
76
|
try {
|
|
77
|
-
const
|
|
78
|
-
|
|
77
|
+
const treeJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}/git/trees/HEAD?recursive=1`);
|
|
78
|
+
const tree = JSON.parse(treeJson);
|
|
79
|
+
const prefix = subdir + '/';
|
|
80
|
+
const mdTreeEntries = (tree.tree || []).filter(f =>
|
|
81
|
+
f.type === 'blob' && f.path.startsWith(prefix) && f.path.endsWith('.md')
|
|
82
|
+
);
|
|
83
|
+
let branch = 'main';
|
|
84
|
+
try {
|
|
85
|
+
const repoJson = await httpsGet(`https://api.github.com/repos/${ownerRepo}`);
|
|
86
|
+
branch = JSON.parse(repoJson).default_branch || 'main';
|
|
87
|
+
} catch {}
|
|
88
|
+
mdFiles = mdTreeEntries.map(f => ({
|
|
89
|
+
name: f.path.replace(prefix, ''),
|
|
90
|
+
download_url: `https://raw.githubusercontent.com/${ownerRepo}/${branch}/${f.path}`
|
|
91
|
+
}));
|
|
79
92
|
} catch (e) {
|
|
80
93
|
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
|
81
94
|
return { ok: false, errors: [`Failed to list ${subdir}/ contents: ${e.message}`], warnings: [] };
|
|
82
95
|
}
|
|
83
|
-
mdFiles = entries.filter(e => e.type === 'file' && e.name.endsWith('.md'));
|
|
84
96
|
} else {
|
|
85
97
|
// No skills subdir — fall back to root-level .md files
|
|
86
98
|
try {
|
|
@@ -131,6 +143,20 @@ async function detectSkillsDirFromAPI(ownerRepo) {
|
|
|
131
143
|
if (dirMap.has(d)) return { subdir: dirMap.get(d), label: d };
|
|
132
144
|
}
|
|
133
145
|
|
|
146
|
+
// 1.5 Nested skills dirs (e.g. .claude/skills, .claude-plugin/commands)
|
|
147
|
+
for (const prefix of ['.claude', '.claude-plugin']) {
|
|
148
|
+
if (dirMap.has(prefix)) {
|
|
149
|
+
for (const d of SKILL_DIRS) {
|
|
150
|
+
const nested = `${prefix}/${d}`;
|
|
151
|
+
try {
|
|
152
|
+
const sub = await httpsGet(`https://api.github.com/repos/${ownerRepo}/contents/${nested}`);
|
|
153
|
+
const subEntries = JSON.parse(sub);
|
|
154
|
+
if (subEntries.length > 0) return { subdir: nested, label: nested };
|
|
155
|
+
} catch {}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
134
160
|
// 2. Any subdir with 2+ .md files — pick the one with most .md files
|
|
135
161
|
const subdirCandidates = entries.filter(e => e.type === 'dir' && !SKIP_DIRS_API.has(e.name.toLowerCase()));
|
|
136
162
|
let best = null, bestCount = 0;
|
|
@@ -152,6 +178,8 @@ const SKIP_DIRS_API = new Set([
|
|
|
152
178
|
'.github', 'docs', 'doc', 'documentation', 'examples', 'example',
|
|
153
179
|
'tests', 'test', 'assets', 'images', 'img', 'media', 'static',
|
|
154
180
|
'node_modules', 'vendor', 'dist', 'build', '.git',
|
|
181
|
+
'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
|
|
182
|
+
'cheatsheets', 'resources',
|
|
155
183
|
]);
|
|
156
184
|
|
|
157
185
|
function git(args, cwd, stdio = 'inherit') {
|
|
@@ -177,16 +205,23 @@ function sparseClone(url, dest, subdir) {
|
|
|
177
205
|
// Try HEAD, then main, then master
|
|
178
206
|
for (const branch of ['HEAD', 'main', 'master']) {
|
|
179
207
|
const r = git(['checkout', branch === 'HEAD' ? 'FETCH_HEAD' : branch], dest, 'pipe');
|
|
180
|
-
if (r.status === 0) return true;
|
|
208
|
+
if (r.status === 0) return finalizeCheckout(dest, true);
|
|
181
209
|
}
|
|
182
210
|
return false;
|
|
183
211
|
}
|
|
184
212
|
|
|
213
|
+
// Shared skip patterns — module scope so both cleanup functions can access them
|
|
214
|
+
const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
|
|
215
|
+
const SKIP_DIRS_LOCAL = new Set([
|
|
216
|
+
'.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots',
|
|
217
|
+
'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor',
|
|
218
|
+
'dist', 'build', 'tests', 'test',
|
|
219
|
+
'references', 'reference', 'refs', 'cheatsheet', 'cheat-sheet',
|
|
220
|
+
'cheatsheets', 'resources',
|
|
221
|
+
]);
|
|
222
|
+
|
|
185
223
|
// After full-clone root: remove files that are not skills and dirs we don't need
|
|
186
224
|
function cleanupRepoRoot(repoRoot) {
|
|
187
|
-
const SKIP_RE = /^(readme|changelog|license|contributing|code.of.conduct|security|authors|credits|install|installation|usage|promotion|faq|glossary|index|overview|summary|roadmap|todo|notes|template|example|sample|demo|guide|tutorial|walkthrough|architecture|design|spec|requirements|privacy|terms|disclaimer|notice|copying|warranty|funding)/i;
|
|
188
|
-
const SKIP_DIRS_LOCAL = new Set(['.github', 'docs', 'doc', 'assets', 'images', 'img', 'screenshots', 'media', 'static', 'scripts', 'ci_scripts', 'node_modules', 'vendor', 'dist', 'build', 'tests', 'test']);
|
|
189
|
-
|
|
190
225
|
let removed = 0;
|
|
191
226
|
const entries = fs.readdirSync(repoRoot, { withFileTypes: true });
|
|
192
227
|
for (const entry of entries) {
|
|
@@ -223,9 +258,14 @@ function cleanupRepoDir(dirPath, SKIP_RE) {
|
|
|
223
258
|
for (const entry of entries) {
|
|
224
259
|
const fullPath = path.join(dirPath, entry.name);
|
|
225
260
|
if (entry.isDirectory()) {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
261
|
+
// Remove entire skip dirs (e.g. references/) nested inside skills dirs
|
|
262
|
+
if (SKIP_DIRS_LOCAL.has(entry.name.toLowerCase())) {
|
|
263
|
+
fs.rmSync(fullPath, { recursive: true, force: true });
|
|
264
|
+
removed++;
|
|
265
|
+
} else {
|
|
266
|
+
removed += cleanupRepoDir(fullPath, SKIP_RE);
|
|
267
|
+
try { if (fs.readdirSync(fullPath).length === 0) fs.rmdirSync(fullPath); } catch {}
|
|
268
|
+
}
|
|
229
269
|
} else if (entry.isFile() && entry.name.endsWith('.md')) {
|
|
230
270
|
const base = entry.name.replace(/\.md$/i, '').toLowerCase();
|
|
231
271
|
if (SKIP_RE.test(base)) {
|
|
@@ -252,28 +292,42 @@ function removeEmptyDirs(dirPath) {
|
|
|
252
292
|
}
|
|
253
293
|
}
|
|
254
294
|
|
|
255
|
-
//
|
|
295
|
+
// After fetch/checkout: force materialization of all sparse-matched files.
|
|
296
|
+
// partial clone (blob:none) + checkout often skips blob download on Windows.
|
|
297
|
+
function forceMaterialize(dest) {
|
|
298
|
+
git(['checkout', 'HEAD', '--', '.'], dest, 'pipe');
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// Update sparse repo — fetch + checkout
|
|
256
302
|
function sparseUpdate(dest, subdir) {
|
|
257
303
|
const fetch = git(['fetch', '--depth=1', 'origin'], dest);
|
|
258
304
|
if (fetch.status !== 0) return false;
|
|
259
305
|
|
|
260
|
-
// Ensure sparse-checkout still set correctly
|
|
261
306
|
git(['sparse-checkout', 'set', '--no-cone', `${subdir}/*.md`, `${subdir}/**/*.md`], dest, 'pipe');
|
|
262
307
|
|
|
263
|
-
for (const ref of ['origin/
|
|
264
|
-
const r = git(['
|
|
265
|
-
if (r.status
|
|
308
|
+
for (const ref of ['origin/main', 'origin/master']) {
|
|
309
|
+
const r = git(['checkout', ref], dest, 'pipe');
|
|
310
|
+
if (r.status !== 0) continue;
|
|
311
|
+
forceMaterialize(dest);
|
|
312
|
+
return true;
|
|
266
313
|
}
|
|
267
314
|
return false;
|
|
268
315
|
}
|
|
269
316
|
|
|
317
|
+
// After checkout, force materialization of sparse-matched files.
|
|
318
|
+
function finalizeCheckout(dest, success) {
|
|
319
|
+
if (success) forceMaterialize(dest);
|
|
320
|
+
return success;
|
|
321
|
+
}
|
|
322
|
+
|
|
270
323
|
// Fallback: clone root but only checkout .md files
|
|
271
324
|
function fullClone(url, dest) {
|
|
272
325
|
if (fs.existsSync(dest)) {
|
|
273
326
|
const fetch = git(['fetch', '--depth=1', '--filter=blob:none', 'origin'], dest);
|
|
274
327
|
if (fetch.status !== 0) return false;
|
|
275
328
|
for (const ref of ['origin/HEAD', 'origin/main', 'origin/master']) {
|
|
276
|
-
|
|
329
|
+
const ok = git(['reset', '--hard', ref], dest, 'pipe').status === 0;
|
|
330
|
+
if (ok) return finalizeCheckout(dest, true);
|
|
277
331
|
}
|
|
278
332
|
return false;
|
|
279
333
|
}
|
|
@@ -287,13 +341,14 @@ function fullClone(url, dest) {
|
|
|
287
341
|
if (fetch.status !== 0) return false;
|
|
288
342
|
for (const branch of ['FETCH_HEAD', 'main', 'master']) {
|
|
289
343
|
const r = git(['checkout', branch], dest, 'pipe');
|
|
290
|
-
if (r.status === 0) return true;
|
|
344
|
+
if (r.status === 0) return finalizeCheckout(dest, true);
|
|
291
345
|
}
|
|
292
346
|
return false;
|
|
293
347
|
}
|
|
294
348
|
|
|
295
|
-
// After clone: detect actual skills dir on disk
|
|
349
|
+
// After clone: detect actual skills dir on disk (flat + nested)
|
|
296
350
|
function detectSkillsDirLocal(repoRoot) {
|
|
351
|
+
// 1. Flat dirs matching SKILL_DIRS (e.g. skills/, commands/)
|
|
297
352
|
for (const dir of SKILL_DIRS) {
|
|
298
353
|
const candidate = path.join(repoRoot, dir);
|
|
299
354
|
if (fs.existsSync(candidate)) {
|
|
@@ -301,6 +356,16 @@ function detectSkillsDirLocal(repoRoot) {
|
|
|
301
356
|
if (files.length >= 1) return { dir: candidate, label: dir, sparse: true };
|
|
302
357
|
}
|
|
303
358
|
}
|
|
359
|
+
// 2. Nested dirs: .claude/skills, .claude-plugin/commands, etc.
|
|
360
|
+
for (const prefix of ['.claude', '.claude-plugin']) {
|
|
361
|
+
for (const dir of SKILL_DIRS) {
|
|
362
|
+
const candidate = path.join(repoRoot, prefix, dir);
|
|
363
|
+
if (fs.existsSync(candidate)) {
|
|
364
|
+
const files = globSync(`${candidate}/**/*.md`);
|
|
365
|
+
if (files.length >= 1) return { dir: candidate, label: `${prefix}/${dir}`, sparse: true };
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
304
369
|
return { dir: repoRoot, label: '(root)', sparse: false };
|
|
305
370
|
}
|
|
306
371
|
|
|
@@ -356,7 +421,17 @@ export async function importFromGitHub(repoUrl) {
|
|
|
356
421
|
const sparseList = isSparse
|
|
357
422
|
? spawnSync('git', ['sparse-checkout', 'list'], { cwd: dest, encoding: 'utf8' }).stdout.trim()
|
|
358
423
|
: '';
|
|
359
|
-
skillsSubdir =
|
|
424
|
+
skillsSubdir = (() => {
|
|
425
|
+
const lines = sparseList.split('\n').map(l => l.trim()).filter(Boolean);
|
|
426
|
+
if (lines.length === 0) return null;
|
|
427
|
+
const exact = lines.find(l => SKILL_DIRS.includes(l));
|
|
428
|
+
if (exact) return exact;
|
|
429
|
+
for (const line of lines) {
|
|
430
|
+
const m = line.match(/^(.+)\/(?:\*\*\/)?\*\.md$/);
|
|
431
|
+
if (m && !m[1].includes('*')) return m[1];
|
|
432
|
+
}
|
|
433
|
+
return null;
|
|
434
|
+
})();
|
|
360
435
|
|
|
361
436
|
cloneOk = skillsSubdir
|
|
362
437
|
? sparseUpdate(dest, skillsSubdir)
|
|
@@ -381,7 +456,12 @@ export async function importFromGitHub(repoUrl) {
|
|
|
381
456
|
removeEmptyDirs(dest);
|
|
382
457
|
}
|
|
383
458
|
|
|
384
|
-
const { dir:
|
|
459
|
+
const { dir: localDir, label: localLabel } = detectSkillsDirLocal(dest);
|
|
460
|
+
// Prefer the known skillsSubdir (from API detection or sparse patterns) as the
|
|
461
|
+
// canonical skills directory — it's more accurate than local heuristics for
|
|
462
|
+
// repos with non-standard dir names (e.g. "specialized", "cli", or nested paths)
|
|
463
|
+
const skillsDir = skillsSubdir ? path.join(dest, skillsSubdir) : localDir;
|
|
464
|
+
const label = skillsSubdir || localLabel;
|
|
385
465
|
const mdFiles = globSync(`${skillsDir}/**/*.md`);
|
|
386
466
|
|
|
387
467
|
if (skillsSubdir) {
|