promptgraph-mcp 2.9.26 → 2.9.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/indexer.js +18 -6
- package/package.json +1 -1
package/indexer.js
CHANGED
|
@@ -19,7 +19,7 @@ function sanitizePath(filePath) {
|
|
|
19
19
|
return path.resolve(filePath);
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
export async function indexBatch(db, skills, { fast = false, silent = false } = {}) {
|
|
22
|
+
export async function indexBatch(db, skills, { fast = false, silent = false, onEmbed = null } = {}) {
|
|
23
23
|
const upsertSkill = db.prepare(`
|
|
24
24
|
INSERT INTO skills (id, name, description, path, source, content, hash, version, author, license, updated_at, downloads, verified)
|
|
25
25
|
VALUES (@id, @name, @description, @path, @source, @content, @hash, @version, @author, @license, @updated_at, @downloads, @verified)
|
|
@@ -72,7 +72,9 @@ export async function indexBatch(db, skills, { fast = false, silent = false } =
|
|
|
72
72
|
const total = texts.length;
|
|
73
73
|
if (!silent) process.stdout.write('\n ⠋ Preparing model...\n');
|
|
74
74
|
let embedStart = null;
|
|
75
|
-
const embeddings = await embedBatch(texts,
|
|
75
|
+
const embeddings = await embedBatch(texts, (done, tot) => {
|
|
76
|
+
if (onEmbed) { onEmbed(done, tot); return; }
|
|
77
|
+
if (silent) return;
|
|
76
78
|
if (!embedStart) {
|
|
77
79
|
process.stdout.write('\x1b[1A\x1b[2K');
|
|
78
80
|
embedStart = Date.now();
|
|
@@ -209,14 +211,23 @@ export async function indexAll({ fast = false } = {}) {
|
|
|
209
211
|
if (!isSkillFile(file, raw)) { skipped++; count++; continue; }
|
|
210
212
|
|
|
211
213
|
if (!processedStart) processedStart = Date.now();
|
|
214
|
+
count++;
|
|
215
|
+
if (count % 50 === 0) {
|
|
216
|
+
const elapsed = Date.now() - start;
|
|
217
|
+
const eta = processedCount > 0 && processedStart
|
|
218
|
+
? Math.round((total - count) * (Date.now() - processedStart) / processedCount / 1000)
|
|
219
|
+
: elapsed > 500 ? Math.round((total - count) * elapsed / count / 1000) : '?';
|
|
220
|
+
progress(count, total, { skipped, eta, errors });
|
|
221
|
+
}
|
|
212
222
|
const parsed = parseSkillFile(file, source, { raw });
|
|
213
223
|
batch.push({ ...parsed, hash });
|
|
214
224
|
|
|
215
225
|
if (batch.length >= BATCH_SIZE) {
|
|
216
226
|
const filtered = await filterWithClassifier(batch);
|
|
217
227
|
classifierRemoved += batch.length - filtered.length;
|
|
218
|
-
await indexBatch(db, filtered, { fast, silent: true
|
|
219
|
-
|
|
228
|
+
await indexBatch(db, filtered, { fast, silent: true, onEmbed: (done, tot) => {
|
|
229
|
+
process.stdout.write(`\r ⠋ Embedding ${done}/${tot} chunks... `);
|
|
230
|
+
}});
|
|
220
231
|
processedCount += filtered.length;
|
|
221
232
|
batch = [];
|
|
222
233
|
const eta = processedCount > 0 ? Math.round((total - count) * (Date.now() - processedStart) / processedCount / 1000) : '?';
|
|
@@ -241,8 +252,9 @@ export async function indexAll({ fast = false } = {}) {
|
|
|
241
252
|
if (batch.length > 0) {
|
|
242
253
|
const filtered = await filterWithClassifier(batch);
|
|
243
254
|
classifierRemoved += batch.length - filtered.length;
|
|
244
|
-
await indexBatch(db, filtered, { fast, silent: true
|
|
245
|
-
|
|
255
|
+
await indexBatch(db, filtered, { fast, silent: true, onEmbed: (done, tot) => {
|
|
256
|
+
process.stdout.write(`\r ⠋ Embedding ${done}/${tot} chunks... `);
|
|
257
|
+
}});
|
|
246
258
|
}
|
|
247
259
|
|
|
248
260
|
progress(total, total, { skipped, errors });
|