pi-knowledge 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +23 -1
- package/index.ts +153 -34
- package/package.json +60 -60
- package/src/diagnostics/health.ts +11 -10
- package/src/embedding/provider.ts +30 -9
- package/src/embedding/vectors.ts +1 -1
- package/src/engine.ts +212 -85
- package/src/indexer/chunker.ts +115 -39
- package/src/indexer/chunkers/code-ast.ts +77 -17
- package/src/search/bm25.ts +19 -10
- package/src/search/fusion.ts +5 -4
- package/src/search/reranker.ts +25 -7
- package/src/search/vector.ts +10 -2
- package/src/storage/sqlite.ts +27 -8
- package/src/watcher/file-watcher.ts +92 -12
package/src/engine.ts
CHANGED
|
@@ -1,18 +1,31 @@
|
|
|
1
1
|
import { existsSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import type Database from "better-sqlite3";
|
|
4
|
-
import {
|
|
4
|
+
import { type DiagnosticResult, diagnoseKB } from "./diagnostics/health.ts";
|
|
5
|
+
import { dispose as disposeEmbedding, embedDocuments, embedQuery } from "./embedding/provider.ts";
|
|
5
6
|
import { loadVectors, saveVectors } from "./embedding/vectors.ts";
|
|
6
|
-
import { chunkFile,
|
|
7
|
+
import { chunkFile, contentHash, preTokenizeForFTS, walkDir } from "./indexer/chunker.ts";
|
|
7
8
|
import { searchBM25 } from "./search/bm25.ts";
|
|
8
9
|
import { reciprocalRankFusion } from "./search/fusion.ts";
|
|
9
|
-
import {
|
|
10
|
+
import { disposeReranker, rerank } from "./search/reranker.ts";
|
|
10
11
|
import { searchVector } from "./search/vector.ts";
|
|
11
|
-
import { diagnoseKB, type DiagnosticResult } from "./diagnostics/health.ts";
|
|
12
12
|
import {
|
|
13
|
-
createKB,
|
|
14
|
-
|
|
13
|
+
createKB,
|
|
14
|
+
deleteChunksByIds,
|
|
15
|
+
deleteKB,
|
|
16
|
+
getChunkById,
|
|
17
|
+
getChunkHashesByKB,
|
|
18
|
+
getChunkIdsByKB,
|
|
19
|
+
getChunksByKB,
|
|
20
|
+
getFileCount,
|
|
21
|
+
getKB,
|
|
22
|
+
getKBByName,
|
|
23
|
+
insertChunks,
|
|
15
24
|
type KnowledgeBase,
|
|
25
|
+
listKBs,
|
|
26
|
+
openDatabase,
|
|
27
|
+
updateKBCounts,
|
|
28
|
+
updateKBStatus,
|
|
16
29
|
} from "./storage/sqlite.ts";
|
|
17
30
|
|
|
18
31
|
export interface SearchOptions {
|
|
@@ -45,6 +58,33 @@ export interface SearchResponse {
|
|
|
45
58
|
|
|
46
59
|
export type ProgressCallback = (msg: string) => void;
|
|
47
60
|
|
|
61
|
+
interface ImportedChunk {
|
|
62
|
+
content: string;
|
|
63
|
+
file_path: string;
|
|
64
|
+
file_type: string;
|
|
65
|
+
start_line: number;
|
|
66
|
+
end_line: number;
|
|
67
|
+
metadata_json?: string;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function chunkUrl(source: string, signal?: AbortSignal): Promise<Awaited<ReturnType<typeof chunkFile>>> {
|
|
71
|
+
const res = await fetch(source, { signal });
|
|
72
|
+
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
73
|
+
const html = await res.text();
|
|
74
|
+
const text = html
|
|
75
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
76
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
77
|
+
.replace(/<[^>]+>/g, " ")
|
|
78
|
+
.replace(/&[a-z]+;/gi, " ")
|
|
79
|
+
.replace(/\s+/g, " ")
|
|
80
|
+
.trim();
|
|
81
|
+
return chunkFile(text, source);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function normalizeExtractedText(text: string | string[]): string {
|
|
85
|
+
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
86
|
+
}
|
|
87
|
+
|
|
48
88
|
export class KnowledgeEngine {
|
|
49
89
|
private db: Database.Database | null = null;
|
|
50
90
|
private knowledgeDir: string = "";
|
|
@@ -57,7 +97,8 @@ export class KnowledgeEngine {
|
|
|
57
97
|
}
|
|
58
98
|
|
|
59
99
|
private getVectors(kbId: string): Float32Array[] {
|
|
60
|
-
|
|
100
|
+
const cached = this.vectorCache.get(kbId);
|
|
101
|
+
if (cached) return cached;
|
|
61
102
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
62
103
|
const vectors = loadVectors(vectorPath);
|
|
63
104
|
this.vectorCache.set(kbId, vectors);
|
|
@@ -68,15 +109,24 @@ export class KnowledgeEngine {
|
|
|
68
109
|
this.vectorCache.delete(kbId);
|
|
69
110
|
}
|
|
70
111
|
|
|
71
|
-
async add(
|
|
112
|
+
async add(
|
|
113
|
+
source: string,
|
|
114
|
+
name: string,
|
|
115
|
+
onProgress?: ProgressCallback,
|
|
116
|
+
signal?: AbortSignal,
|
|
117
|
+
): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
72
118
|
if (!this.db) throw new Error("Engine not initialized");
|
|
73
119
|
const resolvedSource = resolve(source);
|
|
74
120
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
75
121
|
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
76
122
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
77
|
-
const sourceType = isDir ? "directory" : isFile ? "file" : "text";
|
|
123
|
+
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
78
124
|
|
|
79
|
-
const kb = createKB(this.db, {
|
|
125
|
+
const kb = createKB(this.db, {
|
|
126
|
+
name,
|
|
127
|
+
source_path: isDir || isFile ? resolvedSource : isUrl ? source : undefined,
|
|
128
|
+
source_type: sourceType,
|
|
129
|
+
});
|
|
80
130
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
81
131
|
|
|
82
132
|
try {
|
|
@@ -84,20 +134,13 @@ export class KnowledgeEngine {
|
|
|
84
134
|
|
|
85
135
|
if (isUrl) {
|
|
86
136
|
onProgress?.(`Fetching ${source}...`);
|
|
87
|
-
|
|
88
|
-
if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
|
|
89
|
-
const html = await res.text();
|
|
90
|
-
const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
91
|
-
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
92
|
-
.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/gi, " ")
|
|
93
|
-
.replace(/\s+/g, " ").trim();
|
|
94
|
-
allChunks = await chunkFile(text, source);
|
|
137
|
+
allChunks = await chunkUrl(source, signal);
|
|
95
138
|
} else if (isFile && resolvedSource.endsWith(".pdf")) {
|
|
96
139
|
onProgress?.("Extracting text from PDF...");
|
|
97
140
|
const { extractText } = await import("unpdf");
|
|
98
141
|
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
99
142
|
const { text } = await extractText(new Uint8Array(buf));
|
|
100
|
-
allChunks = await chunkFile(text, resolvedSource);
|
|
143
|
+
allChunks = await chunkFile(normalizeExtractedText(text), resolvedSource);
|
|
101
144
|
} else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
102
145
|
onProgress?.("Extracting text from DOCX...");
|
|
103
146
|
const mammoth = await import("mammoth");
|
|
@@ -108,7 +151,7 @@ export class KnowledgeEngine {
|
|
|
108
151
|
const files = walkDir(resolvedSource);
|
|
109
152
|
onProgress?.(`Found ${files.length} files, chunking...`);
|
|
110
153
|
for (const file of files) {
|
|
111
|
-
allChunks.push(...await chunkFile(file.content, file.relPath));
|
|
154
|
+
allChunks.push(...(await chunkFile(file.content, file.relPath)));
|
|
112
155
|
}
|
|
113
156
|
} else if (isFile) {
|
|
114
157
|
const { readFileSync } = await import("node:fs");
|
|
@@ -133,7 +176,9 @@ export class KnowledgeEngine {
|
|
|
133
176
|
updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
|
|
134
177
|
updateKBStatus(this.db, kb.id, "ready");
|
|
135
178
|
|
|
136
|
-
|
|
179
|
+
const savedKB = getKB(this.db, kb.id);
|
|
180
|
+
if (!savedKB) throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
|
|
181
|
+
return { kb: savedKB, chunkCount: allChunks.length };
|
|
137
182
|
} catch (e) {
|
|
138
183
|
// Clean up partial state on failure
|
|
139
184
|
deleteKB(this.db, kb.id);
|
|
@@ -141,11 +186,15 @@ export class KnowledgeEngine {
|
|
|
141
186
|
}
|
|
142
187
|
}
|
|
143
188
|
|
|
144
|
-
async update(
|
|
189
|
+
async update(
|
|
190
|
+
nameOrId: string,
|
|
191
|
+
onProgress?: ProgressCallback,
|
|
192
|
+
signal?: AbortSignal,
|
|
193
|
+
): Promise<{ added: number; removed: number; unchanged: number }> {
|
|
145
194
|
if (!this.db) throw new Error("Engine not initialized");
|
|
146
195
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
147
196
|
if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
148
|
-
if (!kb.source_path || !existsSync(kb.source_path)) {
|
|
197
|
+
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
149
198
|
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
150
199
|
}
|
|
151
200
|
|
|
@@ -154,11 +203,13 @@ export class KnowledgeEngine {
|
|
|
154
203
|
try {
|
|
155
204
|
// 1. Scan and chunk current source
|
|
156
205
|
onProgress?.("Scanning source...");
|
|
157
|
-
const isDir = statSync(kb.source_path).isDirectory();
|
|
158
206
|
let newChunks: Awaited<ReturnType<typeof chunkFile>> = [];
|
|
159
|
-
if (
|
|
207
|
+
if (kb.source_type === "url") {
|
|
208
|
+
onProgress?.(`Fetching ${kb.source_path}...`);
|
|
209
|
+
newChunks = await chunkUrl(kb.source_path, signal);
|
|
210
|
+
} else if (statSync(kb.source_path).isDirectory()) {
|
|
160
211
|
const files = walkDir(kb.source_path);
|
|
161
|
-
for (const file of files) newChunks.push(...await chunkFile(file.content, file.relPath));
|
|
212
|
+
for (const file of files) newChunks.push(...(await chunkFile(file.content, file.relPath)));
|
|
162
213
|
} else {
|
|
163
214
|
const { readFileSync } = await import("node:fs");
|
|
164
215
|
newChunks = await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path);
|
|
@@ -166,7 +217,6 @@ export class KnowledgeEngine {
|
|
|
166
217
|
|
|
167
218
|
// 2. Load existing state: chunks (hash→id) + vectors (ordered)
|
|
168
219
|
const existingHashes = getChunkHashesByKB(this.db, kb.id);
|
|
169
|
-
const existingChunkIds = getChunkIdsByKB(this.db, kb.id);
|
|
170
220
|
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
171
221
|
const existingVectors = loadVectors(vectorPath);
|
|
172
222
|
|
|
@@ -192,7 +242,10 @@ export class KnowledgeEngine {
|
|
|
192
242
|
let newVectors: Float32Array[] = [];
|
|
193
243
|
if (chunksToAdd.length > 0) {
|
|
194
244
|
onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
|
|
195
|
-
newVectors = await embedDocuments(
|
|
245
|
+
newVectors = await embedDocuments(
|
|
246
|
+
chunksToAdd.map((c) => c.content),
|
|
247
|
+
signal,
|
|
248
|
+
);
|
|
196
249
|
insertChunks(this.db, kb.id, chunksToAdd);
|
|
197
250
|
}
|
|
198
251
|
|
|
@@ -203,7 +256,11 @@ export class KnowledgeEngine {
|
|
|
203
256
|
|
|
204
257
|
// 6. Rebuild vector file in DB chunk order (reusing cached vectors)
|
|
205
258
|
const finalChunks = getChunksByKB(this.db, kb.id);
|
|
206
|
-
const finalVectors =
|
|
259
|
+
const finalVectors: Float32Array[] = [];
|
|
260
|
+
for (const chunk of finalChunks) {
|
|
261
|
+
const vector = vectorCache.get(chunk.content_hash);
|
|
262
|
+
if (vector) finalVectors.push(vector);
|
|
263
|
+
}
|
|
207
264
|
saveVectors(vectorPath, finalVectors);
|
|
208
265
|
this.vectorCache.set(kb.id, finalVectors);
|
|
209
266
|
|
|
@@ -220,25 +277,27 @@ export class KnowledgeEngine {
|
|
|
220
277
|
|
|
221
278
|
async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
|
|
222
279
|
if (!this.db) throw new Error("Engine not initialized");
|
|
280
|
+
const db = this.db;
|
|
223
281
|
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
|
|
224
282
|
|
|
225
|
-
const kbs = kb_id ? [getKB(
|
|
283
|
+
const kbs = kb_id ? ([getKB(db, kb_id)].filter(Boolean) as KnowledgeBase[]) : listKBs(db);
|
|
226
284
|
if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
|
|
227
285
|
|
|
228
286
|
const warnings: string[] = [];
|
|
229
|
-
|
|
287
|
+
const allResults: { chunkId: string; score: number }[] = [];
|
|
230
288
|
|
|
231
289
|
for (const kb of kbs) {
|
|
232
290
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
233
|
-
warnings.push(
|
|
291
|
+
warnings.push(
|
|
292
|
+
`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`,
|
|
293
|
+
);
|
|
234
294
|
}
|
|
235
|
-
const chunkIds = getChunkIdsByKB(
|
|
295
|
+
const chunkIds = getChunkIdsByKB(db, kb.id);
|
|
236
296
|
if (chunkIds.length === 0) continue;
|
|
237
297
|
|
|
238
298
|
if (mode === "fast") {
|
|
239
|
-
allResults.push(...searchBM25(
|
|
299
|
+
allResults.push(...searchBM25(db, query, 50, kb.id));
|
|
240
300
|
} else if (mode === "semantic") {
|
|
241
|
-
|
|
242
301
|
const vectors = this.getVectors(kb.id);
|
|
243
302
|
if (vectors.length > 0) {
|
|
244
303
|
const queryVec = await embedQuery(query);
|
|
@@ -246,8 +305,8 @@ export class KnowledgeEngine {
|
|
|
246
305
|
}
|
|
247
306
|
} else {
|
|
248
307
|
// hybrid: BM25 + vector + RRF (both scoped to this KB)
|
|
249
|
-
const bm25Results = searchBM25(
|
|
250
|
-
|
|
308
|
+
const bm25Results = searchBM25(db, query, 50, kb.id);
|
|
309
|
+
|
|
251
310
|
const vectors = this.getVectors(kb.id);
|
|
252
311
|
let vecResults: { chunkId: string; score: number }[] = [];
|
|
253
312
|
if (vectors.length > 0) {
|
|
@@ -261,14 +320,18 @@ export class KnowledgeEngine {
|
|
|
261
320
|
|
|
262
321
|
// Deduplicate and sort
|
|
263
322
|
const seen = new Set<string>();
|
|
264
|
-
const unique = allResults.filter((r) => {
|
|
323
|
+
const unique = allResults.filter((r) => {
|
|
324
|
+
if (seen.has(r.chunkId)) return false;
|
|
325
|
+
seen.add(r.chunkId);
|
|
326
|
+
return true;
|
|
327
|
+
});
|
|
265
328
|
unique.sort((a, b) => b.score - a.score);
|
|
266
329
|
|
|
267
330
|
// Apply metadata filters post-retrieval
|
|
268
331
|
let filtered = unique;
|
|
269
332
|
if (filters?.file_type || filters?.path_pattern) {
|
|
270
333
|
filtered = unique.filter((r) => {
|
|
271
|
-
const chunk = getChunkById(
|
|
334
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
272
335
|
if (!chunk) return false;
|
|
273
336
|
if (filters.file_type && chunk.file_type !== filters.file_type) return false;
|
|
274
337
|
if (filters.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
|
|
@@ -278,39 +341,65 @@ export class KnowledgeEngine {
|
|
|
278
341
|
|
|
279
342
|
// Deep mode: rerank top-20 with cross-encoder
|
|
280
343
|
if (mode === "deep" && filtered.length > 0) {
|
|
281
|
-
const candidates = filtered
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
344
|
+
const candidates = filtered
|
|
345
|
+
.slice(0, 20)
|
|
346
|
+
.map((r) => {
|
|
347
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
348
|
+
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
349
|
+
})
|
|
350
|
+
.filter(Boolean) as Array<{ chunkId: string; content: string }>;
|
|
285
351
|
const reranked = await rerank(query, candidates, limit);
|
|
286
|
-
const results: SearchResult[] =
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
352
|
+
const results: SearchResult[] = [];
|
|
353
|
+
for (const r of reranked) {
|
|
354
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
355
|
+
if (!chunk) continue;
|
|
356
|
+
const kbObj = getKB(db, chunk.kb_id);
|
|
357
|
+
results.push({
|
|
358
|
+
content: chunk.content,
|
|
359
|
+
file_path: chunk.file_path,
|
|
360
|
+
file_type: chunk.file_type,
|
|
361
|
+
kb_name: kbObj?.name ?? "unknown",
|
|
362
|
+
score: r.score,
|
|
363
|
+
snippet: chunk.content.slice(0, 200),
|
|
364
|
+
start_line: chunk.start_line,
|
|
365
|
+
end_line: chunk.end_line,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
return {
|
|
369
|
+
results,
|
|
370
|
+
total_count: results.length,
|
|
371
|
+
has_more: false,
|
|
372
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
373
|
+
};
|
|
292
374
|
}
|
|
293
375
|
|
|
294
376
|
const total = filtered.length;
|
|
295
377
|
const page = filtered.slice(offset, offset + limit);
|
|
296
378
|
|
|
297
|
-
const results: SearchResult[] = page
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
379
|
+
const results: SearchResult[] = page
|
|
380
|
+
.map((r) => {
|
|
381
|
+
const chunk = getChunkById(db, r.chunkId);
|
|
382
|
+
if (!chunk) return null;
|
|
383
|
+
const kb = getKB(db, chunk.kb_id);
|
|
384
|
+
return {
|
|
385
|
+
content: chunk.content,
|
|
386
|
+
file_path: chunk.file_path,
|
|
387
|
+
file_type: chunk.file_type,
|
|
388
|
+
kb_name: kb?.name ?? "unknown",
|
|
389
|
+
score: r.score,
|
|
390
|
+
snippet: chunk.content.slice(0, 200),
|
|
391
|
+
start_line: chunk.start_line,
|
|
392
|
+
end_line: chunk.end_line,
|
|
393
|
+
};
|
|
394
|
+
})
|
|
395
|
+
.filter(Boolean) as SearchResult[];
|
|
396
|
+
|
|
397
|
+
return {
|
|
398
|
+
results,
|
|
399
|
+
total_count: total,
|
|
400
|
+
has_more: offset + limit < total,
|
|
401
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
402
|
+
};
|
|
314
403
|
}
|
|
315
404
|
|
|
316
405
|
remove(nameOrId: string): boolean {
|
|
@@ -335,7 +424,8 @@ export class KnowledgeEngine {
|
|
|
335
424
|
|
|
336
425
|
diagnose(): DiagnosticResult[] {
|
|
337
426
|
if (!this.db) return [];
|
|
338
|
-
|
|
427
|
+
const db = this.db;
|
|
428
|
+
return listKBs(db).map((kb) => diagnoseKB(db, kb));
|
|
339
429
|
}
|
|
340
430
|
|
|
341
431
|
async exportKB(nameOrId: string, outputPath: string): Promise<number> {
|
|
@@ -344,32 +434,69 @@ export class KnowledgeEngine {
|
|
|
344
434
|
if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
345
435
|
const chunks = getChunksByKB(this.db, kb.id);
|
|
346
436
|
const { writeFileSync } = await import("node:fs");
|
|
347
|
-
const header = JSON.stringify({
|
|
348
|
-
|
|
349
|
-
|
|
437
|
+
const header = JSON.stringify({
|
|
438
|
+
name: kb.name,
|
|
439
|
+
description: kb.description,
|
|
440
|
+
source_type: "text",
|
|
441
|
+
chunk_count: chunks.length,
|
|
442
|
+
});
|
|
443
|
+
const lines = [
|
|
444
|
+
header,
|
|
445
|
+
...chunks.map((c) =>
|
|
446
|
+
JSON.stringify({
|
|
447
|
+
content: c.content,
|
|
448
|
+
file_path: c.file_path,
|
|
449
|
+
file_type: c.file_type,
|
|
450
|
+
start_line: c.start_line,
|
|
451
|
+
end_line: c.end_line,
|
|
452
|
+
metadata_json: c.metadata_json,
|
|
453
|
+
}),
|
|
454
|
+
),
|
|
455
|
+
];
|
|
456
|
+
writeFileSync(outputPath, `${lines.join("\n")}\n`);
|
|
350
457
|
return chunks.length;
|
|
351
458
|
}
|
|
352
459
|
|
|
353
|
-
async importKB(
|
|
460
|
+
async importKB(
|
|
461
|
+
inputPath: string,
|
|
462
|
+
onProgress?: ProgressCallback,
|
|
463
|
+
signal?: AbortSignal,
|
|
464
|
+
): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
|
|
354
465
|
if (!this.db) throw new Error("Engine not initialized");
|
|
355
466
|
const { readFileSync } = await import("node:fs");
|
|
356
467
|
const lines = readFileSync(inputPath, "utf-8").trim().split("\n");
|
|
357
468
|
if (lines.length < 1) throw new Error("Empty import file");
|
|
358
|
-
const header = JSON.parse(lines[0]);
|
|
359
|
-
const kb = createKB(this.db, { name: header.name, description: header.description,
|
|
469
|
+
const header = JSON.parse(lines[0]) as { name: string; description?: string };
|
|
470
|
+
const kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
360
471
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
472
|
+
try {
|
|
473
|
+
const chunkData = lines.slice(1).map((l) => JSON.parse(l) as ImportedChunk);
|
|
474
|
+
onProgress?.(`Importing ${chunkData.length} chunks, embedding...`);
|
|
475
|
+
const texts = chunkData.map((c) => c.content);
|
|
476
|
+
const vectors = await embedDocuments(texts, signal);
|
|
477
|
+
const chunks = chunkData.map((c) => ({
|
|
478
|
+
content_hash: contentHash(c.content),
|
|
479
|
+
content: c.content,
|
|
480
|
+
content_tokenized: preTokenizeForFTS(c.content),
|
|
481
|
+
file_path: c.file_path,
|
|
482
|
+
file_type: c.file_type,
|
|
483
|
+
start_line: c.start_line,
|
|
484
|
+
end_line: c.end_line,
|
|
485
|
+
metadata_json: c.metadata_json || "{}",
|
|
486
|
+
}));
|
|
487
|
+
insertChunks(this.db, kb.id, chunks);
|
|
488
|
+
const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
|
|
489
|
+
saveVectors(vectorPath, vectors);
|
|
490
|
+
this.vectorCache.set(kb.id, vectors);
|
|
491
|
+
updateKBCounts(this.db, kb.id, chunks.length, new Set(chunks.map((c) => c.file_path)).size);
|
|
492
|
+
updateKBStatus(this.db, kb.id, "ready");
|
|
493
|
+
const savedKB = getKB(this.db, kb.id);
|
|
494
|
+
if (!savedKB) throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
495
|
+
return { kb: savedKB, chunkCount: chunks.length };
|
|
496
|
+
} catch (e) {
|
|
497
|
+
deleteKB(this.db, kb.id);
|
|
498
|
+
throw e;
|
|
499
|
+
}
|
|
373
500
|
}
|
|
374
501
|
|
|
375
502
|
async dispose(): Promise<void> {
|