promptgraph-mcp 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +205 -205
- package/ann.js +33 -33
- package/api.js +202 -202
- package/bundle-counts.js +111 -111
- package/chunker.js +28 -28
- package/cli.js +115 -115
- package/commands/bundle.js +150 -150
- package/commands/doctor.js +15 -15
- package/commands/import.js +7 -7
- package/commands/init.js +37 -37
- package/commands/marketplace.js +146 -146
- package/commands/reindex.js +10 -10
- package/commands/search.js +55 -55
- package/commands/setup.js +19 -19
- package/commands/status.js +110 -110
- package/commands/train.js +18 -18
- package/commands/update.js +49 -49
- package/commands/validate.js +63 -63
- package/config.js +72 -72
- package/db.js +157 -157
- package/doctor.js +48 -48
- package/embedder.js +54 -54
- package/github-import.js +750 -745
- package/indexer.js +310 -310
- package/package.json +61 -61
- package/parser.js +69 -69
- package/pg-hook.js +70 -70
- package/platform.js +120 -120
- package/search.js +216 -216
- package/src/filter/classifier.js +88 -88
- package/src/filter/hard-filter.js +62 -62
- package/src/filter/train.js +66 -66
- package/src/reranker/reranker.js +92 -92
- package/src/store/flat-store.js +61 -61
- package/src/store/hnsw-store.js +187 -187
- package/src/store/index.js +19 -19
- package/src/store/vector-store.js +9 -9
- package/src/utils/rate-limiter.js +33 -33
- package/tui.js +418 -418
- package/validate-repo-action.js +139 -139
- package/watcher.js +84 -84
package/search.js
CHANGED
|
@@ -1,216 +1,216 @@
|
|
|
1
|
-
import { embed, cosineSimilarity } from './embedder.js';
|
|
2
|
-
import { getDb, blobToVec } from './db.js';
|
|
3
|
-
import { annSearch } from './ann.js';
|
|
4
|
-
import { TRUST_LEVEL_BOOST } from './marketplace.js';
|
|
5
|
-
|
|
6
|
-
function getHybridWeights(query) {
|
|
7
|
-
const hasTechTerms = /[A-Z]|\d/.test(query);
|
|
8
|
-
if (hasTechTerms) return { embedWeight: 0.5, bm25Weight: 0.5 }
|
|
9
|
-
return { embedWeight: 0.7, bm25Weight: 0.3 }
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
function applyRatingBoost(db, id, score) {
|
|
13
|
-
const r = db.prepare('SELECT success, fail FROM ratings WHERE skill_id = ?').get(id);
|
|
14
|
-
let boost = 1.0;
|
|
15
|
-
if (r && (r.success + r.fail) > 3) {
|
|
16
|
-
boost *= (0.85 + 0.15 * (r.success / (r.success + r.fail)));
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// Trust level boost (verified/official/trusted rank higher)
|
|
20
|
-
const re = db.prepare('SELECT trust_level, popularity FROM registry_entries WHERE id = ?').get(id);
|
|
21
|
-
if (re) {
|
|
22
|
-
boost *= (TRUST_LEVEL_BOOST[re.trust_level] || 1.0);
|
|
23
|
-
// Popularity bump: top 20% of skills get +5%, rest unchanged
|
|
24
|
-
if (re.popularity > 0) {
|
|
25
|
-
const top20 = db.prepare('SELECT popularity FROM registry_entries ORDER BY popularity DESC LIMIT 1').get();
|
|
26
|
-
if (top20 && re.popularity >= top20.popularity * 0.2) {
|
|
27
|
-
boost *= 1.05;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
return score * boost;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function normalizeBM25(raw) {
|
|
36
|
-
return Math.max(0, 1 + raw / 10);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async function runEmbeddingSearch(db, queryVec, topK) {
|
|
40
|
-
// Try ANN index first (fast)
|
|
41
|
-
const annResults = await annSearch(queryVec, topK);
|
|
42
|
-
if (annResults && annResults.length > 0) {
|
|
43
|
-
const bestBySkill = new Map();
|
|
44
|
-
for (const r of annResults) {
|
|
45
|
-
const prev = bestBySkill.get(r.skill_id);
|
|
46
|
-
if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
|
|
47
|
-
}
|
|
48
|
-
return [...bestBySkill.entries()]
|
|
49
|
-
.map(([id, score]) => ({ id, score: Math.max(0, score) }))
|
|
50
|
-
.sort((a, b) => b.score - a.score);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Fallback: brute force cosine (used before first reindex)
|
|
54
|
-
const chunks = db.prepare('SELECT skill_id, embedding FROM chunks').all();
|
|
55
|
-
if (chunks.length > 0) {
|
|
56
|
-
const bestBySkill = new Map();
|
|
57
|
-
for (const chunk of chunks) {
|
|
58
|
-
const score = cosineSimilarity(queryVec, blobToVec(chunk.embedding));
|
|
59
|
-
const prev = bestBySkill.get(chunk.skill_id);
|
|
60
|
-
if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
|
|
61
|
-
}
|
|
62
|
-
return [...bestBySkill.entries()]
|
|
63
|
-
.map(([id, score]) => ({ id, score: Math.max(0, score) }))
|
|
64
|
-
.sort((a, b) => b.score - a.score);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
return [];
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function runBM25Search(db, query, topK) {
|
|
71
|
-
try {
|
|
72
|
-
const terms = query.replace(/[^\w\s]/g, ' ').trim().split(/\s+/).filter(Boolean).join(' OR ');
|
|
73
|
-
const rows = db.prepare(
|
|
74
|
-
`SELECT s.id, bm25(skills_fts) AS score FROM skills_fts
|
|
75
|
-
JOIN skills s ON skills_fts.id = s.id
|
|
76
|
-
WHERE skills_fts MATCH ? ORDER BY score LIMIT ?`
|
|
77
|
-
).all(terms, topK);
|
|
78
|
-
return rows.map(r => ({ id: r.id, score: normalizeBM25(r.score) }));
|
|
79
|
-
} catch {
|
|
80
|
-
return [];
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export async function search(query, topK = 5) {
|
|
85
|
-
const db = getDb();
|
|
86
|
-
const { embedWeight, bm25Weight } = getHybridWeights(query);
|
|
87
|
-
const queryVec = await embed(query);
|
|
88
|
-
|
|
89
|
-
const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
|
|
90
|
-
const bm25Results = runBM25Search(db, query, topK * 4);
|
|
91
|
-
|
|
92
|
-
// Fallback: pure BM25 if embeddings unavailable
|
|
93
|
-
if (!embedResults.length) {
|
|
94
|
-
return bm25Results.slice(0, topK)
|
|
95
|
-
.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score)))
|
|
96
|
-
.filter(Boolean);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// Fallback: pure embedding if BM25 returns nothing
|
|
100
|
-
if (!bm25Results.length) {
|
|
101
|
-
return embedResults.slice(0, topK)
|
|
102
|
-
.map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
|
|
103
|
-
.filter(Boolean);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// Hybrid: combine normalized scores from both signals
|
|
107
|
-
const combined = new Map();
|
|
108
|
-
for (const { id, score } of embedResults) {
|
|
109
|
-
combined.set(id, { embedScore: score, bm25Score: 0 });
|
|
110
|
-
}
|
|
111
|
-
for (const { id, score } of bm25Results) {
|
|
112
|
-
const entry = combined.get(id);
|
|
113
|
-
if (entry) {
|
|
114
|
-
entry.bm25Score = score;
|
|
115
|
-
} else {
|
|
116
|
-
combined.set(id, { embedScore: 0, bm25Score: score });
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
const ordered = [...combined.entries()]
|
|
121
|
-
.map(([id, s]) => ({
|
|
122
|
-
id,
|
|
123
|
-
score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
|
|
124
|
-
}))
|
|
125
|
-
.sort((a, b) => b.score - a.score)
|
|
126
|
-
|
|
127
|
-
const rerankerEnabled = process.env.PG_RERANKER !== '0'
|
|
128
|
-
|
|
129
|
-
if (rerankerEnabled) {
|
|
130
|
-
const { Reranker } = await import('./src/reranker/reranker.js')
|
|
131
|
-
const reranker = new Reranker()
|
|
132
|
-
const topN = ordered.slice(0, Math.max(50, topK * 6))
|
|
133
|
-
.map(({ id, score }) => {
|
|
134
|
-
const s = skillWithSnippet(db, id, score)
|
|
135
|
-
return s ? { id, text: s.snippet, score } : null
|
|
136
|
-
})
|
|
137
|
-
.filter(Boolean)
|
|
138
|
-
const reranked = await reranker.rerank(query, topN, topK)
|
|
139
|
-
return reranked.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score))).filter(Boolean)
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
return ordered.slice(0, topK)
|
|
143
|
-
.map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
|
|
144
|
-
.filter(Boolean);
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
function skillWithSnippet(db, id, score) {
|
|
148
|
-
const skill = db.prepare('SELECT id, name, description, path, source, content, version, author, license, updated_at, downloads, verified FROM skills WHERE id = ?').get(id);
|
|
149
|
-
if (!skill) return null;
|
|
150
|
-
const { content, ...rest } = skill;
|
|
151
|
-
const snippet = content?.replace(/^---[\s\S]*?---\n?/, '').trim().slice(0, 400) || '';
|
|
152
|
-
return { ...rest, score, snippet };
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export function getContext(nameOrId) {
|
|
156
|
-
const db = getDb();
|
|
157
|
-
let skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(nameOrId);
|
|
158
|
-
if (!skill) {
|
|
159
|
-
const res = resolveId(db, nameOrId);
|
|
160
|
-
if (res.error) return res;
|
|
161
|
-
skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(res.id);
|
|
162
|
-
}
|
|
163
|
-
if (!skill) return null;
|
|
164
|
-
const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(skill.id).map(r => r.to_skill);
|
|
165
|
-
const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(skill.id).map(r => r.from_skill);
|
|
166
|
-
|
|
167
|
-
return { ...skill, callees, callers };
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function resolveId(db, nameOrId) {
|
|
171
|
-
const byId = db.prepare('SELECT id FROM skills WHERE id = ?').get(nameOrId);
|
|
172
|
-
if (byId) return { id: byId.id };
|
|
173
|
-
const byName = db.prepare('SELECT id FROM skills WHERE name = ?').all(nameOrId);
|
|
174
|
-
if (byName.length === 1) return { id: byName[0].id };
|
|
175
|
-
if (byName.length > 1) {
|
|
176
|
-
const candidates = byName.map(r => r.id).join(', ');
|
|
177
|
-
return { error: `Ambiguous name "${nameOrId}" — multiple skills match. Use a full id: ${candidates}` };
|
|
178
|
-
}
|
|
179
|
-
return { error: `Skill not found: ${nameOrId}` };
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
export function getCallers(nameOrId) {
|
|
183
|
-
const db = getDb();
|
|
184
|
-
const res = resolveId(db, nameOrId);
|
|
185
|
-
if (res.error) return res;
|
|
186
|
-
return db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(res.id).map(r => r.from_skill);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
export function getCallees(nameOrId) {
|
|
190
|
-
const db = getDb();
|
|
191
|
-
const res = resolveId(db, nameOrId);
|
|
192
|
-
if (res.error) return res;
|
|
193
|
-
return db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(res.id).map(r => r.to_skill);
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
export function getImpact(nameOrId) {
|
|
197
|
-
const db = getDb();
|
|
198
|
-
const res = resolveId(db, nameOrId);
|
|
199
|
-
if (res.error) return res;
|
|
200
|
-
const visited = new Set();
|
|
201
|
-
const queue = [res.id];
|
|
202
|
-
while (queue.length) {
|
|
203
|
-
const cur = queue.shift();
|
|
204
|
-
if (visited.has(cur)) continue;
|
|
205
|
-
visited.add(cur);
|
|
206
|
-
const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(cur).map(r => r.from_skill);
|
|
207
|
-
queue.push(...callers);
|
|
208
|
-
}
|
|
209
|
-
visited.delete(res.id);
|
|
210
|
-
return [...visited];
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
export function listAll() {
|
|
214
|
-
const db = getDb();
|
|
215
|
-
return db.prepare('SELECT id, name, description, source, version, author, license, updated_at, downloads, verified FROM skills ORDER BY source, name').all();
|
|
216
|
-
}
|
|
1
|
+
import { embed, cosineSimilarity } from './embedder.js';
|
|
2
|
+
import { getDb, blobToVec } from './db.js';
|
|
3
|
+
import { annSearch } from './ann.js';
|
|
4
|
+
import { TRUST_LEVEL_BOOST } from './marketplace.js';
|
|
5
|
+
|
|
6
|
+
function getHybridWeights(query) {
|
|
7
|
+
const hasTechTerms = /[A-Z]|\d/.test(query);
|
|
8
|
+
if (hasTechTerms) return { embedWeight: 0.5, bm25Weight: 0.5 }
|
|
9
|
+
return { embedWeight: 0.7, bm25Weight: 0.3 }
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function applyRatingBoost(db, id, score) {
|
|
13
|
+
const r = db.prepare('SELECT success, fail FROM ratings WHERE skill_id = ?').get(id);
|
|
14
|
+
let boost = 1.0;
|
|
15
|
+
if (r && (r.success + r.fail) > 3) {
|
|
16
|
+
boost *= (0.85 + 0.15 * (r.success / (r.success + r.fail)));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Trust level boost (verified/official/trusted rank higher)
|
|
20
|
+
const re = db.prepare('SELECT trust_level, popularity FROM registry_entries WHERE id = ?').get(id);
|
|
21
|
+
if (re) {
|
|
22
|
+
boost *= (TRUST_LEVEL_BOOST[re.trust_level] || 1.0);
|
|
23
|
+
// Popularity bump: top 20% of skills get +5%, rest unchanged
|
|
24
|
+
if (re.popularity > 0) {
|
|
25
|
+
const top20 = db.prepare('SELECT popularity FROM registry_entries ORDER BY popularity DESC LIMIT 1').get();
|
|
26
|
+
if (top20 && re.popularity >= top20.popularity * 0.2) {
|
|
27
|
+
boost *= 1.05;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return score * boost;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function normalizeBM25(raw) {
|
|
36
|
+
return Math.max(0, 1 + raw / 10);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function runEmbeddingSearch(db, queryVec, topK) {
|
|
40
|
+
// Try ANN index first (fast)
|
|
41
|
+
const annResults = await annSearch(queryVec, topK);
|
|
42
|
+
if (annResults && annResults.length > 0) {
|
|
43
|
+
const bestBySkill = new Map();
|
|
44
|
+
for (const r of annResults) {
|
|
45
|
+
const prev = bestBySkill.get(r.skill_id);
|
|
46
|
+
if (!prev || r.score > prev) bestBySkill.set(r.skill_id, r.score);
|
|
47
|
+
}
|
|
48
|
+
return [...bestBySkill.entries()]
|
|
49
|
+
.map(([id, score]) => ({ id, score: Math.max(0, score) }))
|
|
50
|
+
.sort((a, b) => b.score - a.score);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Fallback: brute force cosine (used before first reindex)
|
|
54
|
+
const chunks = db.prepare('SELECT skill_id, embedding FROM chunks').all();
|
|
55
|
+
if (chunks.length > 0) {
|
|
56
|
+
const bestBySkill = new Map();
|
|
57
|
+
for (const chunk of chunks) {
|
|
58
|
+
const score = cosineSimilarity(queryVec, blobToVec(chunk.embedding));
|
|
59
|
+
const prev = bestBySkill.get(chunk.skill_id);
|
|
60
|
+
if (!prev || score > prev) bestBySkill.set(chunk.skill_id, score);
|
|
61
|
+
}
|
|
62
|
+
return [...bestBySkill.entries()]
|
|
63
|
+
.map(([id, score]) => ({ id, score: Math.max(0, score) }))
|
|
64
|
+
.sort((a, b) => b.score - a.score);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return [];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function runBM25Search(db, query, topK) {
|
|
71
|
+
try {
|
|
72
|
+
const terms = query.replace(/[^\w\s]/g, ' ').trim().split(/\s+/).filter(Boolean).join(' OR ');
|
|
73
|
+
const rows = db.prepare(
|
|
74
|
+
`SELECT s.id, bm25(skills_fts) AS score FROM skills_fts
|
|
75
|
+
JOIN skills s ON skills_fts.id = s.id
|
|
76
|
+
WHERE skills_fts MATCH ? ORDER BY score LIMIT ?`
|
|
77
|
+
).all(terms, topK);
|
|
78
|
+
return rows.map(r => ({ id: r.id, score: normalizeBM25(r.score) }));
|
|
79
|
+
} catch {
|
|
80
|
+
return [];
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export async function search(query, topK = 5) {
|
|
85
|
+
const db = getDb();
|
|
86
|
+
const { embedWeight, bm25Weight } = getHybridWeights(query);
|
|
87
|
+
const queryVec = await embed(query);
|
|
88
|
+
|
|
89
|
+
const embedResults = await runEmbeddingSearch(db, queryVec, topK * 4);
|
|
90
|
+
const bm25Results = runBM25Search(db, query, topK * 4);
|
|
91
|
+
|
|
92
|
+
// Fallback: pure BM25 if embeddings unavailable
|
|
93
|
+
if (!embedResults.length) {
|
|
94
|
+
return bm25Results.slice(0, topK)
|
|
95
|
+
.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score)))
|
|
96
|
+
.filter(Boolean);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Fallback: pure embedding if BM25 returns nothing
|
|
100
|
+
if (!bm25Results.length) {
|
|
101
|
+
return embedResults.slice(0, topK)
|
|
102
|
+
.map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
|
|
103
|
+
.filter(Boolean);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Hybrid: combine normalized scores from both signals
|
|
107
|
+
const combined = new Map();
|
|
108
|
+
for (const { id, score } of embedResults) {
|
|
109
|
+
combined.set(id, { embedScore: score, bm25Score: 0 });
|
|
110
|
+
}
|
|
111
|
+
for (const { id, score } of bm25Results) {
|
|
112
|
+
const entry = combined.get(id);
|
|
113
|
+
if (entry) {
|
|
114
|
+
entry.bm25Score = score;
|
|
115
|
+
} else {
|
|
116
|
+
combined.set(id, { embedScore: 0, bm25Score: score });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const ordered = [...combined.entries()]
|
|
121
|
+
.map(([id, s]) => ({
|
|
122
|
+
id,
|
|
123
|
+
score: embedWeight * s.embedScore + bm25Weight * s.bm25Score,
|
|
124
|
+
}))
|
|
125
|
+
.sort((a, b) => b.score - a.score)
|
|
126
|
+
|
|
127
|
+
const rerankerEnabled = process.env.PG_RERANKER !== '0'
|
|
128
|
+
|
|
129
|
+
if (rerankerEnabled) {
|
|
130
|
+
const { Reranker } = await import('./src/reranker/reranker.js')
|
|
131
|
+
const reranker = new Reranker()
|
|
132
|
+
const topN = ordered.slice(0, Math.max(50, topK * 6))
|
|
133
|
+
.map(({ id, score }) => {
|
|
134
|
+
const s = skillWithSnippet(db, id, score)
|
|
135
|
+
return s ? { id, text: s.snippet, score } : null
|
|
136
|
+
})
|
|
137
|
+
.filter(Boolean)
|
|
138
|
+
const reranked = await reranker.rerank(query, topN, topK)
|
|
139
|
+
return reranked.map(r => skillWithSnippet(db, r.id, applyRatingBoost(db, r.id, r.score))).filter(Boolean)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return ordered.slice(0, topK)
|
|
143
|
+
.map(({ id, score }) => skillWithSnippet(db, id, applyRatingBoost(db, id, score)))
|
|
144
|
+
.filter(Boolean);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function skillWithSnippet(db, id, score) {
|
|
148
|
+
const skill = db.prepare('SELECT id, name, description, path, source, content, version, author, license, updated_at, downloads, verified FROM skills WHERE id = ?').get(id);
|
|
149
|
+
if (!skill) return null;
|
|
150
|
+
const { content, ...rest } = skill;
|
|
151
|
+
const snippet = content?.replace(/^---[\s\S]*?---\n?/, '').trim().slice(0, 400) || '';
|
|
152
|
+
return { ...rest, score, snippet };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function getContext(nameOrId) {
|
|
156
|
+
const db = getDb();
|
|
157
|
+
let skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(nameOrId);
|
|
158
|
+
if (!skill) {
|
|
159
|
+
const res = resolveId(db, nameOrId);
|
|
160
|
+
if (res.error) return res;
|
|
161
|
+
skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(res.id);
|
|
162
|
+
}
|
|
163
|
+
if (!skill) return null;
|
|
164
|
+
const callees = db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(skill.id).map(r => r.to_skill);
|
|
165
|
+
const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(skill.id).map(r => r.from_skill);
|
|
166
|
+
|
|
167
|
+
return { ...skill, callees, callers };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function resolveId(db, nameOrId) {
|
|
171
|
+
const byId = db.prepare('SELECT id FROM skills WHERE id = ?').get(nameOrId);
|
|
172
|
+
if (byId) return { id: byId.id };
|
|
173
|
+
const byName = db.prepare('SELECT id FROM skills WHERE name = ?').all(nameOrId);
|
|
174
|
+
if (byName.length === 1) return { id: byName[0].id };
|
|
175
|
+
if (byName.length > 1) {
|
|
176
|
+
const candidates = byName.map(r => r.id).join(', ');
|
|
177
|
+
return { error: `Ambiguous name "${nameOrId}" — multiple skills match. Use a full id: ${candidates}` };
|
|
178
|
+
}
|
|
179
|
+
return { error: `Skill not found: ${nameOrId}` };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function getCallers(nameOrId) {
|
|
183
|
+
const db = getDb();
|
|
184
|
+
const res = resolveId(db, nameOrId);
|
|
185
|
+
if (res.error) return res;
|
|
186
|
+
return db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(res.id).map(r => r.from_skill);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function getCallees(nameOrId) {
|
|
190
|
+
const db = getDb();
|
|
191
|
+
const res = resolveId(db, nameOrId);
|
|
192
|
+
if (res.error) return res;
|
|
193
|
+
return db.prepare('SELECT to_skill FROM edges WHERE from_skill = ?').all(res.id).map(r => r.to_skill);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function getImpact(nameOrId) {
|
|
197
|
+
const db = getDb();
|
|
198
|
+
const res = resolveId(db, nameOrId);
|
|
199
|
+
if (res.error) return res;
|
|
200
|
+
const visited = new Set();
|
|
201
|
+
const queue = [res.id];
|
|
202
|
+
while (queue.length) {
|
|
203
|
+
const cur = queue.shift();
|
|
204
|
+
if (visited.has(cur)) continue;
|
|
205
|
+
visited.add(cur);
|
|
206
|
+
const callers = db.prepare('SELECT from_skill FROM edges WHERE to_skill = ?').all(cur).map(r => r.from_skill);
|
|
207
|
+
queue.push(...callers);
|
|
208
|
+
}
|
|
209
|
+
visited.delete(res.id);
|
|
210
|
+
return [...visited];
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function listAll() {
|
|
214
|
+
const db = getDb();
|
|
215
|
+
return db.prepare('SELECT id, name, description, source, version, author, license, updated_at, downloads, verified FROM skills ORDER BY source, name').all();
|
|
216
|
+
}
|
package/src/filter/classifier.js
CHANGED
|
@@ -1,88 +1,88 @@
|
|
|
1
|
-
import { cosineSimilarity } from '../../embedder.js';
|
|
2
|
-
|
|
3
|
-
const SKILL_THRESHOLD = 0.35;
|
|
4
|
-
const UNSURE_THRESHOLD = 0.15;
|
|
5
|
-
|
|
6
|
-
const DOC_FIRST_HEADERS = /^(overview|introduction|about|background|welcome|getting started|what is|why |table of contents|toc|foreword|preface|readme)/i;
|
|
7
|
-
const INSTRUCTION_HEADERS = /^#{1,3}\s+(steps?|usage|instructions?|how\s+to|when\s+to\s+use|workflow|process|procedure|example|examples?|commands?|output|result)/i;
|
|
8
|
-
const IMPERATIVE_HEADERS = /\b(run|use|apply|execute|check|debug|fix|create|add|remove|deploy|test|write|generate|analyze|review|refactor|optimize|configure|setup|install|scan|audit|validate|search|find|extract|parse)\b/i;
|
|
9
|
-
|
|
10
|
-
export function getFeatureVector(raw) {
|
|
11
|
-
const lines = raw.split('\n');
|
|
12
|
-
const nonEmpty = lines.filter(l => l.trim());
|
|
13
|
-
const headers = nonEmpty.filter(l => /^#{1,3}\s/.test(l));
|
|
14
|
-
const paragraphs = raw.split(/\n\n+/).filter(p => p.trim() && !p.trim().startsWith('#'));
|
|
15
|
-
const words = raw.toLowerCase().split(/\s+/).filter(w => w.length > 3);
|
|
16
|
-
|
|
17
|
-
const firstHeader = headers[0]?.replace(/^#+\s*/, '') || '';
|
|
18
|
-
const longProse = paragraphs.filter(p => p.split(' ').length > 60 && !/```/.test(p));
|
|
19
|
-
const wordRatio = words.length > 80 ? new Set(words).size / words.length : 1;
|
|
20
|
-
|
|
21
|
-
return [
|
|
22
|
-
Number(raw.length >= 150),
|
|
23
|
-
Number(headers.length >= 1),
|
|
24
|
-
Number(lines.some(l => INSTRUCTION_HEADERS.test(l))),
|
|
25
|
-
Number(headers.some(h => IMPERATIVE_HEADERS.test(h))),
|
|
26
|
-
Number(raw.includes('```') || raw.includes(' ')),
|
|
27
|
-
Number(nonEmpty.some(l => /^\d+\.\s/.test(l))),
|
|
28
|
-
Number(nonEmpty.some(l => /^[-*+]\s/.test(l))),
|
|
29
|
-
Number(headers.length >= 2),
|
|
30
|
-
Number(headers.length >= 4),
|
|
31
|
-
Number(DOC_FIRST_HEADERS.test(firstHeader)) ? 1 : 0,
|
|
32
|
-
Number(longProse.length > paragraphs.length * 0.6 && paragraphs.length > 3) ? 1 : 0,
|
|
33
|
-
Number(wordRatio >= 0.22) ? 0 : 1,
|
|
34
|
-
Number(raw.length < 150) ? 1 : 0,
|
|
35
|
-
Number(headers.length < 1) ? 1 : 0,
|
|
36
|
-
];
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Rule A override: if centroid says reject/unsure but file clearly looks like
|
|
41
|
-
* a skill by content (code + instructions + list + verbs, no SKILL in name/path),
|
|
42
|
-
* override to skill. Catches adversarial wrong-heading cases with 0 FP cost.
|
|
43
|
-
*/
|
|
44
|
-
function ruleA(rawVec, raw, filePath) {
|
|
45
|
-
const fv = getFeatureVector(raw);
|
|
46
|
-
const hasCode = fv[4];
|
|
47
|
-
const hasInstructions = fv[2];
|
|
48
|
-
const hasNumberedList = fv[5];
|
|
49
|
-
const hasVerb = fv[3];
|
|
50
|
-
const filename = filePath ? filePath.split(/[/\\]/).pop().replace(/\.md$/i, '').toUpperCase() : '';
|
|
51
|
-
const pathUpper = filePath ? filePath.toUpperCase() : '';
|
|
52
|
-
const hasSkillName = filename.includes('SKILL');
|
|
53
|
-
const hasSkillPath = /SKILL/.test(pathUpper);
|
|
54
|
-
|
|
55
|
-
if (hasCode && hasInstructions && hasNumberedList && hasVerb && !hasSkillName && !hasSkillPath) {
|
|
56
|
-
return { label: 'skill', score: 0.6, method: 'ruleA' };
|
|
57
|
-
}
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export function classify(rawVec, centroids, raw, filePath) {
|
|
62
|
-
if (!centroids) {
|
|
63
|
-
return { label: 'skill', score: 1, method: 'fallback' };
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const goodSim = centroids.good ? cosineSimilarity(rawVec, centroids.good) : 0;
|
|
67
|
-
const badSim = centroids.bad ? cosineSimilarity(rawVec, centroids.bad) : 0;
|
|
68
|
-
|
|
69
|
-
const rawScore = goodSim - badSim;
|
|
70
|
-
const score = (rawScore + 1) / 2;
|
|
71
|
-
|
|
72
|
-
if (score >= SKILL_THRESHOLD) {
|
|
73
|
-
return { label: 'skill', score, goodSim, badSim, method: 'centroid' };
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// Below threshold — try Rule A content override
|
|
77
|
-
if (raw && filePath) {
|
|
78
|
-
const override = ruleA(rawVec, raw, filePath);
|
|
79
|
-
if (override) return override;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
if (score >= UNSURE_THRESHOLD) {
|
|
83
|
-
return { label: 'unsure', score, goodSim, badSim, method: 'centroid' };
|
|
84
|
-
}
|
|
85
|
-
return { label: 'reject', score, goodSim, badSim, method: 'centroid' };
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export { SKILL_THRESHOLD, UNSURE_THRESHOLD };
|
|
1
|
+
import { cosineSimilarity } from '../../embedder.js';
|
|
2
|
+
|
|
3
|
+
const SKILL_THRESHOLD = 0.35;
|
|
4
|
+
const UNSURE_THRESHOLD = 0.15;
|
|
5
|
+
|
|
6
|
+
const DOC_FIRST_HEADERS = /^(overview|introduction|about|background|welcome|getting started|what is|why |table of contents|toc|foreword|preface|readme)/i;
|
|
7
|
+
const INSTRUCTION_HEADERS = /^#{1,3}\s+(steps?|usage|instructions?|how\s+to|when\s+to\s+use|workflow|process|procedure|example|examples?|commands?|output|result)/i;
|
|
8
|
+
const IMPERATIVE_HEADERS = /\b(run|use|apply|execute|check|debug|fix|create|add|remove|deploy|test|write|generate|analyze|review|refactor|optimize|configure|setup|install|scan|audit|validate|search|find|extract|parse)\b/i;
|
|
9
|
+
|
|
10
|
+
export function getFeatureVector(raw) {
|
|
11
|
+
const lines = raw.split('\n');
|
|
12
|
+
const nonEmpty = lines.filter(l => l.trim());
|
|
13
|
+
const headers = nonEmpty.filter(l => /^#{1,3}\s/.test(l));
|
|
14
|
+
const paragraphs = raw.split(/\n\n+/).filter(p => p.trim() && !p.trim().startsWith('#'));
|
|
15
|
+
const words = raw.toLowerCase().split(/\s+/).filter(w => w.length > 3);
|
|
16
|
+
|
|
17
|
+
const firstHeader = headers[0]?.replace(/^#+\s*/, '') || '';
|
|
18
|
+
const longProse = paragraphs.filter(p => p.split(' ').length > 60 && !/```/.test(p));
|
|
19
|
+
const wordRatio = words.length > 80 ? new Set(words).size / words.length : 1;
|
|
20
|
+
|
|
21
|
+
return [
|
|
22
|
+
Number(raw.length >= 150),
|
|
23
|
+
Number(headers.length >= 1),
|
|
24
|
+
Number(lines.some(l => INSTRUCTION_HEADERS.test(l))),
|
|
25
|
+
Number(headers.some(h => IMPERATIVE_HEADERS.test(h))),
|
|
26
|
+
Number(raw.includes('```') || raw.includes(' ')),
|
|
27
|
+
Number(nonEmpty.some(l => /^\d+\.\s/.test(l))),
|
|
28
|
+
Number(nonEmpty.some(l => /^[-*+]\s/.test(l))),
|
|
29
|
+
Number(headers.length >= 2),
|
|
30
|
+
Number(headers.length >= 4),
|
|
31
|
+
Number(DOC_FIRST_HEADERS.test(firstHeader)) ? 1 : 0,
|
|
32
|
+
Number(longProse.length > paragraphs.length * 0.6 && paragraphs.length > 3) ? 1 : 0,
|
|
33
|
+
Number(wordRatio >= 0.22) ? 0 : 1,
|
|
34
|
+
Number(raw.length < 150) ? 1 : 0,
|
|
35
|
+
Number(headers.length < 1) ? 1 : 0,
|
|
36
|
+
];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Rule A override: if centroid says reject/unsure but file clearly looks like
|
|
41
|
+
* a skill by content (code + instructions + list + verbs, no SKILL in name/path),
|
|
42
|
+
* override to skill. Catches adversarial wrong-heading cases with 0 FP cost.
|
|
43
|
+
*/
|
|
44
|
+
function ruleA(rawVec, raw, filePath) {
|
|
45
|
+
const fv = getFeatureVector(raw);
|
|
46
|
+
const hasCode = fv[4];
|
|
47
|
+
const hasInstructions = fv[2];
|
|
48
|
+
const hasNumberedList = fv[5];
|
|
49
|
+
const hasVerb = fv[3];
|
|
50
|
+
const filename = filePath ? filePath.split(/[/\\]/).pop().replace(/\.md$/i, '').toUpperCase() : '';
|
|
51
|
+
const pathUpper = filePath ? filePath.toUpperCase() : '';
|
|
52
|
+
const hasSkillName = filename.includes('SKILL');
|
|
53
|
+
const hasSkillPath = /SKILL/.test(pathUpper);
|
|
54
|
+
|
|
55
|
+
if (hasCode && hasInstructions && hasNumberedList && hasVerb && !hasSkillName && !hasSkillPath) {
|
|
56
|
+
return { label: 'skill', score: 0.6, method: 'ruleA' };
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function classify(rawVec, centroids, raw, filePath) {
|
|
62
|
+
if (!centroids) {
|
|
63
|
+
return { label: 'skill', score: 1, method: 'fallback' };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const goodSim = centroids.good ? cosineSimilarity(rawVec, centroids.good) : 0;
|
|
67
|
+
const badSim = centroids.bad ? cosineSimilarity(rawVec, centroids.bad) : 0;
|
|
68
|
+
|
|
69
|
+
const rawScore = goodSim - badSim;
|
|
70
|
+
const score = (rawScore + 1) / 2;
|
|
71
|
+
|
|
72
|
+
if (score >= SKILL_THRESHOLD) {
|
|
73
|
+
return { label: 'skill', score, goodSim, badSim, method: 'centroid' };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Below threshold — try Rule A content override
|
|
77
|
+
if (raw && filePath) {
|
|
78
|
+
const override = ruleA(rawVec, raw, filePath);
|
|
79
|
+
if (override) return override;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (score >= UNSURE_THRESHOLD) {
|
|
83
|
+
return { label: 'unsure', score, goodSim, badSim, method: 'centroid' };
|
|
84
|
+
}
|
|
85
|
+
return { label: 'reject', score, goodSim, badSim, method: 'centroid' };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export { SKILL_THRESHOLD, UNSURE_THRESHOLD };
|