pi-knowledge 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +5 -0
- package/dist/src/engine.js +52 -3
- package/docs/configuration.md +9 -0
- package/package.json +9 -12
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.10.1] - 2026-09-10
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Repaired `knowledge_update` recovery for knowledge bases whose SQLite chunks are ahead of a short or truncated vector file; update now rebuilds or re-embeds missing vectors instead of leaving the KB in `error` (#13).
|
|
7
|
+
- Aligned the published Tree-sitter runtime and grammar dependency graph so clean npm installs no longer emit peer override warnings for `tree-sitter@0.25.1` incompatibilities (#14).
|
|
8
|
+
|
|
3
9
|
## [0.10.0] - 2026-08-27
|
|
4
10
|
|
|
5
11
|
### Added
|
package/README.md
CHANGED
|
@@ -83,6 +83,11 @@ pi install npm:pi-knowledge
|
|
|
83
83
|
# Install for OMP
|
|
84
84
|
omp install npm:pi-knowledge
|
|
85
85
|
|
|
86
|
+
# On minimal Ubuntu/Debian servers, install native build tools first because
|
|
87
|
+
# OMP's Bun-based npm install compiles better-sqlite3 from source.
|
|
88
|
+
sudo apt-get update && sudo apt-get install -y build-essential python3
|
|
89
|
+
omp install npm:pi-knowledge
|
|
90
|
+
|
|
86
91
|
# Or from source
|
|
87
92
|
pi install ./pi-knowledge
|
|
88
93
|
omp install ./pi-knowledge
|
package/dist/src/engine.js
CHANGED
|
@@ -23,6 +23,11 @@ function isCancellationError(error) {
|
|
|
23
23
|
function tempVectorPath(vectorPath) {
|
|
24
24
|
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
25
25
|
}
|
|
26
|
+
function assertEmbeddingBatchSize(vectors, expected, operation) {
|
|
27
|
+
if (vectors.length !== expected) {
|
|
28
|
+
throw new Error(`Embedding provider returned ${vectors.length} vectors for ${expected} ${operation} chunks`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
26
31
|
function retrievalModeFor(mode) {
|
|
27
32
|
if (mode === "auto")
|
|
28
33
|
return "hybrid";
|
|
@@ -759,6 +764,7 @@ export class KnowledgeEngine {
|
|
|
759
764
|
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
760
765
|
if (signal?.aborted)
|
|
761
766
|
throw new Error("Cancelled");
|
|
767
|
+
assertEmbeddingBatchSize(vectors, batch.length, "add");
|
|
762
768
|
persistEmbeddingMetadata(db, kb.id, embeddingConfig, vectors);
|
|
763
769
|
insertChunks(db, kb.id, batch);
|
|
764
770
|
writer.append(vectors);
|
|
@@ -923,7 +929,7 @@ export class KnowledgeEngine {
|
|
|
923
929
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
924
930
|
const embeddingModel = embeddingConfigLabel(embeddingConfig);
|
|
925
931
|
const currentSignature = kb.embedding_dimension === null ? undefined : embeddingSignature(embeddingConfig, kb.embedding_dimension);
|
|
926
|
-
|
|
932
|
+
let canReuseExistingVectors = kb.embedding_signature !== null && kb.embedding_dimension !== null && kb.embedding_signature === currentSignature;
|
|
927
933
|
let replacementVectorPath;
|
|
928
934
|
let addedVectorPath;
|
|
929
935
|
let addedVectorWriter;
|
|
@@ -943,6 +949,24 @@ export class KnowledgeEngine {
|
|
|
943
949
|
existingHashes.set(chunk.content_hash, entries);
|
|
944
950
|
existingIndex++;
|
|
945
951
|
}
|
|
952
|
+
if (canReuseExistingVectors) {
|
|
953
|
+
const oldVectorReader = openVectorReader(vectorPath);
|
|
954
|
+
try {
|
|
955
|
+
if (!oldVectorReader ||
|
|
956
|
+
oldVectorReader.count !== existingIndex ||
|
|
957
|
+
oldVectorReader.dim !== kb.embedding_dimension) {
|
|
958
|
+
canReuseExistingVectors = false;
|
|
959
|
+
const vectorCount = oldVectorReader?.count ?? 0;
|
|
960
|
+
const vectorDimension = oldVectorReader?.dim ?? 0;
|
|
961
|
+
const message = `Stored vector file for "${kb.name}" is incomplete or incompatible (${vectorCount} vectors/${vectorDimension}d for ${existingIndex} chunks/${kb.embedding_dimension}d); rebuilding all vectors`;
|
|
962
|
+
updateIndexingJob(this.db, kb.id, { phase: "embedding", message });
|
|
963
|
+
onProgress?.(message);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
finally {
|
|
967
|
+
oldVectorReader?.close();
|
|
968
|
+
}
|
|
969
|
+
}
|
|
946
970
|
const reusableHashes = canReuseExistingVectors
|
|
947
971
|
? existingHashes
|
|
948
972
|
: new Map();
|
|
@@ -983,6 +1007,7 @@ export class KnowledgeEngine {
|
|
|
983
1007
|
const newVectors = await embedDocuments(batch.map((c) => buildChunkEmbeddingText(c)), signal);
|
|
984
1008
|
if (signal?.aborted)
|
|
985
1009
|
throw new Error("Cancelled");
|
|
1010
|
+
assertEmbeddingBatchSize(newVectors, batch.length, "update");
|
|
986
1011
|
addedVectorWriter.append(newVectors);
|
|
987
1012
|
for (let i = 0; i < batch.length; i++) {
|
|
988
1013
|
const indexes = newVectorIndexByHash.get(batch[i].content_hash) ?? [];
|
|
@@ -1122,6 +1147,29 @@ export class KnowledgeEngine {
|
|
|
1122
1147
|
const indexes = indexesByHash.get(hash);
|
|
1123
1148
|
return indexes?.shift();
|
|
1124
1149
|
};
|
|
1150
|
+
const repairMissingVector = async (chunk) => {
|
|
1151
|
+
const message = `Re-embedding chunk with missing stored vector: ${chunk.id}`;
|
|
1152
|
+
if (!this.db)
|
|
1153
|
+
throw new Error("Engine not initialized");
|
|
1154
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1155
|
+
phase: "embedding",
|
|
1156
|
+
message,
|
|
1157
|
+
processed_files: scannedFiles,
|
|
1158
|
+
processed_chunks: finalChunkCount,
|
|
1159
|
+
added_chunks: addedCount,
|
|
1160
|
+
removed_chunks: idsToRemove.length,
|
|
1161
|
+
unchanged_chunks: unchanged,
|
|
1162
|
+
});
|
|
1163
|
+
onProgress?.(message);
|
|
1164
|
+
const repairedVectors = await embedDocuments([buildChunkEmbeddingText(chunk)], signal);
|
|
1165
|
+
if (signal?.aborted)
|
|
1166
|
+
throw new Error("Cancelled");
|
|
1167
|
+
assertEmbeddingBatchSize(repairedVectors, 1, "repair");
|
|
1168
|
+
const [repairedVector] = repairedVectors;
|
|
1169
|
+
if (!repairedVector)
|
|
1170
|
+
throw new Error(`Embedding provider returned no repaired vector for chunk: ${chunk.id}`);
|
|
1171
|
+
return repairedVector;
|
|
1172
|
+
};
|
|
1125
1173
|
try {
|
|
1126
1174
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1127
1175
|
if (signal?.aborted)
|
|
@@ -1130,13 +1178,13 @@ export class KnowledgeEngine {
|
|
|
1130
1178
|
continue;
|
|
1131
1179
|
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
1132
1180
|
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
1133
|
-
|
|
1181
|
+
let vector = oldVectorReader && oldVectorIndex !== undefined
|
|
1134
1182
|
? oldVectorReader.read(oldVectorIndex)
|
|
1135
1183
|
: newVectorReader && newVectorIndex !== undefined
|
|
1136
1184
|
? newVectorReader.read(newVectorIndex)
|
|
1137
1185
|
: undefined;
|
|
1138
1186
|
if (!vector)
|
|
1139
|
-
|
|
1187
|
+
vector = await repairMissingVector(chunk);
|
|
1140
1188
|
finalEmbeddingDimension ??= vector.length;
|
|
1141
1189
|
vectorWriter.append([vector]);
|
|
1142
1190
|
finalChunkCount++;
|
|
@@ -1816,6 +1864,7 @@ export class KnowledgeEngine {
|
|
|
1816
1864
|
onProgress?.(message);
|
|
1817
1865
|
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1818
1866
|
throwIfAborted(signal);
|
|
1867
|
+
assertEmbeddingBatchSize(vectors, batch.length, "import");
|
|
1819
1868
|
persistEmbeddingMetadata(this.db, kb.id, embeddingConfig, vectors);
|
|
1820
1869
|
insertChunks(this.db, kb.id, batch);
|
|
1821
1870
|
insertSymbols(this.db, kb.id, symbols);
|
package/docs/configuration.md
CHANGED
|
@@ -103,6 +103,15 @@ When `PI_KNOWLEDGE_E2E_PDF` or `PI_KNOWLEDGE_E2E_DOCX` is unset, `npm run test:e
|
|
|
103
103
|
|
|
104
104
|
`pi-knowledge` supports direct Pi extension loading through `pi -e ./extension.js` and OMP-compatible loading through the same packaged entry shim. The root `extension.js` and `index.ts` stay startup-light so install-time validation can inspect the extension without resolving native runtime dependencies such as `better-sqlite3` or `onnxruntime-node`.
|
|
105
105
|
|
|
106
|
+
Minimal Ubuntu/Debian servers need native build tools before `omp install npm:pi-knowledge`,
|
|
107
|
+
because OMP's Bun-based npm install compiles `better-sqlite3` from source when Bun cannot use
|
|
108
|
+
the package prebuild:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
sudo apt-get update && sudo apt-get install -y build-essential python3
|
|
112
|
+
omp install npm:pi-knowledge
|
|
113
|
+
```
|
|
114
|
+
|
|
106
115
|
Compatibility guarantees:
|
|
107
116
|
|
|
108
117
|
- The packaged `extension.js` loads built `dist/index.js` when present and falls back to source `index.ts` for local development.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -54,22 +54,19 @@
|
|
|
54
54
|
"better-sqlite3": "12.11.1",
|
|
55
55
|
"ignore": "7.0.5",
|
|
56
56
|
"mammoth": "1.12.0",
|
|
57
|
-
"tree-sitter": "0.
|
|
58
|
-
"tree-sitter-bash": "0.
|
|
59
|
-
"tree-sitter-c": "0.
|
|
60
|
-
"tree-sitter-cpp": "0.23.
|
|
61
|
-
"tree-sitter-go": "0.
|
|
57
|
+
"tree-sitter": "0.21.1",
|
|
58
|
+
"tree-sitter-bash": "0.21.0",
|
|
59
|
+
"tree-sitter-c": "0.21.4",
|
|
60
|
+
"tree-sitter-cpp": "0.23.2",
|
|
61
|
+
"tree-sitter-go": "0.21.2",
|
|
62
62
|
"tree-sitter-java": "0.23.5",
|
|
63
|
-
"tree-sitter-javascript": "0.
|
|
64
|
-
"tree-sitter-python": "0.
|
|
63
|
+
"tree-sitter-javascript": "0.23.1",
|
|
64
|
+
"tree-sitter-python": "0.21.0",
|
|
65
65
|
"tree-sitter-qmljs": "0.3.1",
|
|
66
|
-
"tree-sitter-rust": "0.
|
|
66
|
+
"tree-sitter-rust": "0.21.0",
|
|
67
67
|
"tree-sitter-typescript": "0.23.2",
|
|
68
68
|
"unpdf": "1.6.2"
|
|
69
69
|
},
|
|
70
|
-
"overrides": {
|
|
71
|
-
"tree-sitter": "0.25.1"
|
|
72
|
-
},
|
|
73
70
|
"devDependencies": {
|
|
74
71
|
"@biomejs/biome": "2.3.5",
|
|
75
72
|
"@types/better-sqlite3": "7.6.13",
|