promptgraph-mcp 2.9.12 → 2.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/indexer.js +22 -15
- package/package.json +1 -1
package/indexer.js
CHANGED
|
@@ -53,15 +53,7 @@ export async function indexBatch(db, skills, { fast = false } = {}) {
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
57
|
-
if (!fast && allChunks.length) {
|
|
58
|
-
const texts = allChunks.map(c => c.text);
|
|
59
|
-
process.stdout.write(` Embedding ${texts.length} chunks...`);
|
|
60
|
-
embeddings = await embedBatch(texts);
|
|
61
|
-
process.stdout.write('\r' + ' '.repeat(40) + '\r');
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// pass 1: upsert all skills + chunks (no edges yet)
|
|
56
|
+
// pass 1: upsert all skills metadata first (so chunks can reference them)
|
|
65
57
|
db.transaction(() => {
|
|
66
58
|
for (const skill of skills) {
|
|
67
59
|
const id = skillId(skill.source, skill.name);
|
|
@@ -72,14 +64,29 @@ export async function indexBatch(db, skills, { fast = false } = {}) {
|
|
|
72
64
|
deleteEdges.run(id);
|
|
73
65
|
}
|
|
74
66
|
}
|
|
75
|
-
if (!fast) {
|
|
76
|
-
for (let i = 0; i < allChunks.length; i++) {
|
|
77
|
-
const { id, chunkIndex, text } = allChunks[i];
|
|
78
|
-
upsertChunk.run(id, chunkIndex, text, vecToBlob(embeddings[i]));
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
67
|
})();
|
|
82
68
|
|
|
69
|
+
// pass 1b: embed in batches of 50 with progress bar, save each batch immediately
|
|
70
|
+
if (!fast && allChunks.length) {
|
|
71
|
+
const EMBED_BATCH = 50;
|
|
72
|
+
const total = allChunks.length;
|
|
73
|
+
for (let i = 0; i < total; i += EMBED_BATCH) {
|
|
74
|
+
const batch = allChunks.slice(i, i + EMBED_BATCH);
|
|
75
|
+
const texts = batch.map(c => c.text);
|
|
76
|
+
const vecs = await embedBatch(texts);
|
|
77
|
+
db.transaction(() => {
|
|
78
|
+
for (let j = 0; j < batch.length; j++) {
|
|
79
|
+
const { id, chunkIndex, text } = batch[j];
|
|
80
|
+
upsertChunk.run(id, chunkIndex, text, vecToBlob(vecs[j]));
|
|
81
|
+
}
|
|
82
|
+
})();
|
|
83
|
+
const done = Math.min(i + EMBED_BATCH, total);
|
|
84
|
+
const pct = Math.round(done / total * 100);
|
|
85
|
+
process.stdout.write(`\r Embedding chunks... ${done}/${total} (${pct}%) `);
|
|
86
|
+
}
|
|
87
|
+
process.stdout.write('\r' + ' '.repeat(50) + '\r');
|
|
88
|
+
}
|
|
89
|
+
|
|
83
90
|
// pass 2: resolve edges after all skills in batch are committed
|
|
84
91
|
const resolveSameSource = db.prepare("SELECT id FROM skills WHERE name = ? AND source = ? LIMIT 1");
|
|
85
92
|
const resolveAny = db.prepare("SELECT id FROM skills WHERE name = ? ORDER BY id LIMIT 1");
|