pi-knowledge 0.4.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +19 -12
- package/dist/index.js +186 -42
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +13 -12
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +61 -10
- package/dist/src/engine.js +591 -186
- package/dist/src/indexer/chunker.d.ts +3 -1
- package/dist/src/indexer/chunker.js +57 -48
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +159 -2
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +10 -2
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
4
|
import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
|
-
import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile,
|
|
6
|
+
import { addSkippedScanEntry, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
|
+
import { extractSymbols } from "./indexer/symbols.js";
|
|
7
8
|
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
8
9
|
import { searchBM25 } from "./search/bm25.js";
|
|
9
10
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
|
11
12
|
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
12
13
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
13
14
|
import { searchVectorFile } from "./search/vector.js";
|
|
14
|
-
import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile,
|
|
15
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
|
|
15
16
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
16
17
|
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
17
18
|
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
@@ -24,6 +25,26 @@ function isCancellationError(error) {
|
|
|
24
25
|
function tempVectorPath(vectorPath) {
|
|
25
26
|
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
26
27
|
}
|
|
28
|
+
function retrievalModeFor(mode) {
|
|
29
|
+
if (mode === "auto")
|
|
30
|
+
return "hybrid";
|
|
31
|
+
if (mode === "code" || mode === "config" || mode === "errors")
|
|
32
|
+
return "fast";
|
|
33
|
+
if (mode === "docs" || mode === "decision")
|
|
34
|
+
return "hybrid";
|
|
35
|
+
return mode;
|
|
36
|
+
}
|
|
37
|
+
function matchReasonFor(mode) {
|
|
38
|
+
if (mode === "deep")
|
|
39
|
+
return "rerank";
|
|
40
|
+
if (mode === "adaptive")
|
|
41
|
+
return "adaptive";
|
|
42
|
+
if (mode === "semantic")
|
|
43
|
+
return "vector";
|
|
44
|
+
if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
|
|
45
|
+
return "bm25";
|
|
46
|
+
return "hybrid";
|
|
47
|
+
}
|
|
27
48
|
function toScanOptions(options = {}) {
|
|
28
49
|
return {
|
|
29
50
|
includeSuggestedText: options.include_suggested_text === true,
|
|
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
|
|
|
60
81
|
return {};
|
|
61
82
|
}
|
|
62
83
|
}
|
|
63
|
-
function planDirectoryScan(dirPath, options = {}) {
|
|
84
|
+
function planDirectoryScan(dirPath, options = {}, signal) {
|
|
64
85
|
const skipped = createSkippedScanStats();
|
|
65
86
|
let files = 0;
|
|
66
87
|
let bytes = 0;
|
|
67
88
|
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
89
|
+
throwIfAborted(signal);
|
|
68
90
|
files++;
|
|
69
91
|
bytes += file.size;
|
|
70
92
|
}
|
|
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
|
|
|
76
98
|
skipped,
|
|
77
99
|
};
|
|
78
100
|
}
|
|
101
|
+
function sourceMtimeFor(kb, filePath) {
|
|
102
|
+
if (!kb?.source_path)
|
|
103
|
+
return undefined;
|
|
104
|
+
const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
|
|
105
|
+
try {
|
|
106
|
+
return statSync(absPath).mtimeMs;
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function kbTrustMultiplier(kb) {
|
|
113
|
+
let multiplier = kb.status === "stale" ? 0.92 : 1;
|
|
114
|
+
if (kb.source_type === "directory" || kb.source_type === "file")
|
|
115
|
+
multiplier *= 1.04;
|
|
116
|
+
if (kb.source_type === "text" || kb.source_type === "url")
|
|
117
|
+
multiplier *= 0.98;
|
|
118
|
+
return multiplier;
|
|
119
|
+
}
|
|
79
120
|
function formatBytes(bytes) {
|
|
80
121
|
if (bytes < 1024)
|
|
81
122
|
return `${bytes} B`;
|
|
@@ -347,22 +388,181 @@ async function chunkUrl(source, signal) {
|
|
|
347
388
|
function normalizeExtractedText(text) {
|
|
348
389
|
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
349
390
|
}
|
|
391
|
+
async function extractSourceFileContent(filePath, signal) {
|
|
392
|
+
const lowerPath = filePath.toLowerCase();
|
|
393
|
+
if (lowerPath.endsWith(".pdf")) {
|
|
394
|
+
throwIfAborted(signal);
|
|
395
|
+
// PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
396
|
+
const { extractText } = await import("unpdf");
|
|
397
|
+
throwIfAborted(signal);
|
|
398
|
+
const buf = readFileSync(filePath);
|
|
399
|
+
throwIfAborted(signal);
|
|
400
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
401
|
+
throwIfAborted(signal);
|
|
402
|
+
return { content: normalizeExtractedText(text), fileType: "pdf" };
|
|
403
|
+
}
|
|
404
|
+
if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
|
|
405
|
+
throwIfAborted(signal);
|
|
406
|
+
// DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
407
|
+
const mammoth = await import("mammoth");
|
|
408
|
+
throwIfAborted(signal);
|
|
409
|
+
const result = await mammoth.extractRawText({ path: filePath });
|
|
410
|
+
throwIfAborted(signal);
|
|
411
|
+
return { content: result.value, fileType: "docx" };
|
|
412
|
+
}
|
|
413
|
+
if (!isReadableTextFile(filePath)) {
|
|
414
|
+
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
415
|
+
}
|
|
416
|
+
throwIfAborted(signal);
|
|
417
|
+
return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
|
|
418
|
+
}
|
|
419
|
+
function looksLikeLocalPath(source) {
|
|
420
|
+
const trimmed = source.trim();
|
|
421
|
+
if (!trimmed || trimmed.includes("\n") || trimmed.includes("\r"))
|
|
422
|
+
return false;
|
|
423
|
+
if (/^(?:https?:)?\/\//.test(trimmed))
|
|
424
|
+
return false;
|
|
425
|
+
if (trimmed.startsWith(".") || trimmed.startsWith("/") || trimmed.startsWith("~"))
|
|
426
|
+
return true;
|
|
427
|
+
if (trimmed.includes("/") || trimmed.includes("\\"))
|
|
428
|
+
return true;
|
|
429
|
+
return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
|
|
430
|
+
}
|
|
431
|
+
function classifySource(source) {
|
|
432
|
+
const resolvedSource = resolve(source);
|
|
433
|
+
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
434
|
+
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
435
|
+
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
436
|
+
if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
|
|
437
|
+
throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
|
|
438
|
+
}
|
|
439
|
+
return {
|
|
440
|
+
resolvedSource,
|
|
441
|
+
isUrl,
|
|
442
|
+
isDir,
|
|
443
|
+
isFile,
|
|
444
|
+
sourceType: isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text",
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
async function extractScannableFileContent(file, signal) {
|
|
448
|
+
return extractSourceFileContent(file.path, signal);
|
|
449
|
+
}
|
|
450
|
+
async function extractScannableFileContentOrSkip(file, skipped, signal) {
|
|
451
|
+
try {
|
|
452
|
+
return await extractScannableFileContent(file, signal);
|
|
453
|
+
}
|
|
454
|
+
catch (error) {
|
|
455
|
+
if (signal?.aborted || isCancellationError(error))
|
|
456
|
+
throw error;
|
|
457
|
+
addSkippedScanEntry(skipped, { path: file.relPath, reason: "extraction_failed", size: file.size });
|
|
458
|
+
return undefined;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
function throwIfAborted(signal) {
|
|
462
|
+
if (signal?.aborted)
|
|
463
|
+
throw new Error("Cancelled");
|
|
464
|
+
}
|
|
465
|
+
function importedChunkToInsert(chunk) {
|
|
466
|
+
const metadataJson = chunk.metadata_json || "{}";
|
|
467
|
+
return {
|
|
468
|
+
content_hash: chunkIdentityHash({
|
|
469
|
+
content: chunk.content,
|
|
470
|
+
filePath: chunk.file_path,
|
|
471
|
+
fileType: chunk.file_type,
|
|
472
|
+
startLine: chunk.start_line,
|
|
473
|
+
endLine: chunk.end_line,
|
|
474
|
+
metadataJson,
|
|
475
|
+
}),
|
|
476
|
+
content: chunk.content,
|
|
477
|
+
content_tokenized: "",
|
|
478
|
+
file_path: chunk.file_path,
|
|
479
|
+
file_type: chunk.file_type,
|
|
480
|
+
start_line: chunk.start_line,
|
|
481
|
+
end_line: chunk.end_line,
|
|
482
|
+
metadata_json: metadataJson,
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
async function writeLine(stream, line) {
|
|
486
|
+
if (stream.write(`${line}\n`))
|
|
487
|
+
return;
|
|
488
|
+
await new Promise((resolve, reject) => {
|
|
489
|
+
const cleanup = () => {
|
|
490
|
+
stream.off("drain", onDrain);
|
|
491
|
+
stream.off("error", onError);
|
|
492
|
+
};
|
|
493
|
+
const onDrain = () => {
|
|
494
|
+
cleanup();
|
|
495
|
+
resolve();
|
|
496
|
+
};
|
|
497
|
+
const onError = (error) => {
|
|
498
|
+
cleanup();
|
|
499
|
+
reject(error);
|
|
500
|
+
};
|
|
501
|
+
stream.once("drain", onDrain);
|
|
502
|
+
stream.once("error", onError);
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
async function finishWriteStream(stream) {
|
|
506
|
+
await new Promise((resolve, reject) => {
|
|
507
|
+
const cleanup = () => {
|
|
508
|
+
stream.off("finish", onFinish);
|
|
509
|
+
stream.off("error", onError);
|
|
510
|
+
};
|
|
511
|
+
const onFinish = () => {
|
|
512
|
+
cleanup();
|
|
513
|
+
resolve();
|
|
514
|
+
};
|
|
515
|
+
const onError = (error) => {
|
|
516
|
+
cleanup();
|
|
517
|
+
reject(error);
|
|
518
|
+
};
|
|
519
|
+
stream.once("finish", onFinish);
|
|
520
|
+
stream.once("error", onError);
|
|
521
|
+
stream.end();
|
|
522
|
+
});
|
|
523
|
+
}
|
|
350
524
|
export class KnowledgeEngine {
|
|
351
525
|
db = null;
|
|
352
526
|
knowledgeDir = "";
|
|
353
527
|
activeUpdates = new Map();
|
|
528
|
+
activeMutations = new Map();
|
|
529
|
+
disposing = false;
|
|
530
|
+
runExclusive(keys, description, operation) {
|
|
531
|
+
if (this.disposing)
|
|
532
|
+
throw new Error("Knowledge engine is shutting down");
|
|
533
|
+
const mutationKeys = Array.isArray(keys) ? keys : [keys];
|
|
534
|
+
for (const key of mutationKeys) {
|
|
535
|
+
if (this.activeMutations.has(key))
|
|
536
|
+
throw new Error(`${description} is already running`);
|
|
537
|
+
}
|
|
538
|
+
const run = Promise.resolve()
|
|
539
|
+
.then(operation)
|
|
540
|
+
.finally(() => {
|
|
541
|
+
for (const key of mutationKeys) {
|
|
542
|
+
if (this.activeMutations.get(key) === run)
|
|
543
|
+
this.activeMutations.delete(key);
|
|
544
|
+
}
|
|
545
|
+
});
|
|
546
|
+
for (const key of mutationKeys)
|
|
547
|
+
this.activeMutations.set(key, run);
|
|
548
|
+
return run;
|
|
549
|
+
}
|
|
354
550
|
async initialize(knowledgeDir) {
|
|
355
551
|
this.knowledgeDir = knowledgeDir;
|
|
356
552
|
this.db = openDatabase(knowledgeDir);
|
|
553
|
+
this.disposing = false;
|
|
554
|
+
}
|
|
555
|
+
vectorPathFor(kbId) {
|
|
556
|
+
return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
357
557
|
}
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
const
|
|
558
|
+
deleteVectorFile(kbId) {
|
|
559
|
+
rmSync(this.vectorPathFor(kbId), { force: true });
|
|
560
|
+
}
|
|
561
|
+
plan(source, options = {}, signal) {
|
|
562
|
+
throwIfAborted(signal);
|
|
563
|
+
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
|
|
364
564
|
if (isDir) {
|
|
365
|
-
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
565
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
|
|
366
566
|
return {
|
|
367
567
|
source_type: "directory",
|
|
368
568
|
scannable_files: plan.files,
|
|
@@ -373,7 +573,7 @@ export class KnowledgeEngine {
|
|
|
373
573
|
}
|
|
374
574
|
if (isFile) {
|
|
375
575
|
const skipped = createSkippedScanStats();
|
|
376
|
-
const supportedDocument =
|
|
576
|
+
const supportedDocument = isSupportedDocumentFile(resolvedSource);
|
|
377
577
|
if (!supportedDocument && !isReadableTextFile(resolvedSource)) {
|
|
378
578
|
skipped.total = 1;
|
|
379
579
|
skipped.by_reason.binary = 1;
|
|
@@ -405,14 +605,15 @@ export class KnowledgeEngine {
|
|
|
405
605
|
};
|
|
406
606
|
}
|
|
407
607
|
async add(source, name, onProgress, signal, options = {}) {
|
|
608
|
+
throwIfAborted(signal);
|
|
609
|
+
return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
|
|
610
|
+
}
|
|
611
|
+
async addUnlocked(source, name, onProgress, signal, options = {}) {
|
|
408
612
|
if (!this.db)
|
|
409
613
|
throw new Error("Engine not initialized");
|
|
614
|
+
throwIfAborted(signal);
|
|
410
615
|
const db = this.db;
|
|
411
|
-
const resolvedSource =
|
|
412
|
-
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
413
|
-
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
414
|
-
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
415
|
-
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
616
|
+
const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
|
|
416
617
|
const scanOptions = toScanOptions(options);
|
|
417
618
|
const existingKB = getKBByName(db, name);
|
|
418
619
|
if (existingKB) {
|
|
@@ -429,7 +630,7 @@ export class KnowledgeEngine {
|
|
|
429
630
|
let vectorWriter;
|
|
430
631
|
let tempVectorFile;
|
|
431
632
|
try {
|
|
432
|
-
const vectorPath =
|
|
633
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
433
634
|
tempVectorFile = tempVectorPath(vectorPath);
|
|
434
635
|
const writer = openVectorWriter(tempVectorFile);
|
|
435
636
|
vectorWriter = writer;
|
|
@@ -438,7 +639,8 @@ export class KnowledgeEngine {
|
|
|
438
639
|
const pendingChunks = [];
|
|
439
640
|
const startedAt = Date.now();
|
|
440
641
|
let latestSkippedTotal = 0;
|
|
441
|
-
|
|
642
|
+
let latestSkippedSummary = "none";
|
|
643
|
+
const reportProgress = (phase, processedFiles, totalFiles, skippedTotal = latestSkippedTotal) => {
|
|
442
644
|
latestSkippedTotal = skippedTotal;
|
|
443
645
|
const elapsed = Date.now() - startedAt;
|
|
444
646
|
const chunkRate = chunkCount / Math.max(1, elapsed / 1000);
|
|
@@ -488,33 +690,26 @@ export class KnowledgeEngine {
|
|
|
488
690
|
await flushPending(processedFiles, totalFiles);
|
|
489
691
|
}
|
|
490
692
|
};
|
|
693
|
+
const addSymbols = (content, filePath, fileType) => {
|
|
694
|
+
insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
|
|
695
|
+
};
|
|
491
696
|
if (isUrl) {
|
|
492
697
|
const message = `Fetching ${source}...`;
|
|
493
698
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
494
699
|
onProgress?.(message);
|
|
495
700
|
const chunks = await chunkUrl(source, signal);
|
|
496
701
|
fileCount = 1;
|
|
702
|
+
addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
|
|
497
703
|
await addChunks(chunks);
|
|
498
704
|
}
|
|
499
|
-
else if (isFile
|
|
500
|
-
|
|
501
|
-
onProgress?.("Extracting text from PDF...");
|
|
502
|
-
const { extractText } = await import("unpdf");
|
|
503
|
-
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
504
|
-
const { text } = await extractText(new Uint8Array(buf));
|
|
505
|
-
fileCount = 1;
|
|
506
|
-
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
507
|
-
}
|
|
508
|
-
else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
509
|
-
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
510
|
-
onProgress?.("Extracting text from DOCX...");
|
|
511
|
-
const mammoth = await import("mammoth");
|
|
512
|
-
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
705
|
+
else if (isFile) {
|
|
706
|
+
const extracted = await extractSourceFileContent(resolvedSource, signal);
|
|
513
707
|
fileCount = 1;
|
|
514
|
-
|
|
708
|
+
addSymbols(extracted.content, resolvedSource, extracted.fileType);
|
|
709
|
+
await addChunks(await chunkFile(extracted.content, resolvedSource));
|
|
515
710
|
}
|
|
516
711
|
else if (isDir) {
|
|
517
|
-
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
712
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
|
|
518
713
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
519
714
|
updateIndexingJob(this.db, kb.id, {
|
|
520
715
|
phase: "planning",
|
|
@@ -533,19 +728,27 @@ export class KnowledgeEngine {
|
|
|
533
728
|
onProgress?.(scanningMessage);
|
|
534
729
|
const skipped = createSkippedScanStats();
|
|
535
730
|
let processedFiles = 0;
|
|
536
|
-
for (const file of
|
|
731
|
+
for (const file of iterateScannableFiles(resolvedSource, skipped, scanOptions)) {
|
|
537
732
|
if (signal?.aborted)
|
|
538
733
|
throw new Error("Cancelled");
|
|
539
|
-
const
|
|
734
|
+
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
735
|
+
if (!extracted) {
|
|
736
|
+
latestSkippedTotal = skipped.total;
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
const chunks = await chunkFile(extracted.content, file.relPath);
|
|
540
740
|
processedFiles++;
|
|
541
741
|
latestSkippedTotal = skipped.total;
|
|
542
742
|
if (chunks.length > 0)
|
|
543
743
|
fileCount++;
|
|
744
|
+
addSymbols(extracted.content, file.relPath, extracted.fileType);
|
|
544
745
|
await addChunks(chunks, processedFiles, plan.files);
|
|
545
746
|
if (processedFiles % 25 === 0)
|
|
546
747
|
reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
547
748
|
}
|
|
548
|
-
|
|
749
|
+
latestSkippedTotal = skipped.total;
|
|
750
|
+
latestSkippedSummary = summarizeSkippedScan(skipped);
|
|
751
|
+
const finalizingMessage = `Scanned ${processedFiles} files, skipped ${skipped.total} (${latestSkippedSummary}), finalizing...`;
|
|
549
752
|
updateIndexingJob(this.db, kb.id, {
|
|
550
753
|
phase: "finalizing",
|
|
551
754
|
message: finalizingMessage,
|
|
@@ -557,17 +760,9 @@ export class KnowledgeEngine {
|
|
|
557
760
|
});
|
|
558
761
|
onProgress?.(finalizingMessage);
|
|
559
762
|
}
|
|
560
|
-
else if (isFile) {
|
|
561
|
-
if (!isReadableTextFile(resolvedSource)) {
|
|
562
|
-
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
563
|
-
}
|
|
564
|
-
const { readFileSync } = await import("node:fs");
|
|
565
|
-
const content = readFileSync(resolvedSource, "utf-8");
|
|
566
|
-
fileCount = 1;
|
|
567
|
-
await addChunks(await chunkFile(content, resolvedSource));
|
|
568
|
-
}
|
|
569
763
|
else {
|
|
570
764
|
fileCount = 1;
|
|
765
|
+
addSymbols(source, "inline-text", "text");
|
|
571
766
|
await addChunks(await chunkFile(source, "inline-text"));
|
|
572
767
|
}
|
|
573
768
|
await flushPending();
|
|
@@ -581,7 +776,8 @@ export class KnowledgeEngine {
|
|
|
581
776
|
const savedKB = getKB(this.db, kb.id);
|
|
582
777
|
if (!savedKB)
|
|
583
778
|
throw new Error(`Knowledge base disappeared after add: ${kb.id}`);
|
|
584
|
-
const
|
|
779
|
+
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
780
|
+
const readyMessage = `Ready: ${chunkCount} chunks from ${savedFileCount} files in ${formatDuration(Date.now() - startedAt)}${skippedReadySuffix}`;
|
|
585
781
|
updateIndexingJob(this.db, kb.id, {
|
|
586
782
|
phase: "ready",
|
|
587
783
|
message: readyMessage,
|
|
@@ -600,15 +796,18 @@ export class KnowledgeEngine {
|
|
|
600
796
|
rmSync(tempVectorFile, { force: true });
|
|
601
797
|
if (this.db) {
|
|
602
798
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
|
|
799
|
+
deleteKB(this.db, kb.id);
|
|
800
|
+
this.deleteVectorFile(kb.id);
|
|
603
801
|
}
|
|
604
|
-
// Clean up partial state on failure
|
|
605
|
-
deleteKB(this.db, kb.id);
|
|
606
802
|
throw e;
|
|
607
803
|
}
|
|
608
804
|
}
|
|
609
805
|
async update(nameOrId, onProgress, signal) {
|
|
610
806
|
if (!this.db)
|
|
611
807
|
throw new Error("Engine not initialized");
|
|
808
|
+
if (this.disposing)
|
|
809
|
+
throw new Error("Knowledge engine is shutting down");
|
|
810
|
+
throwIfAborted(signal);
|
|
612
811
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
613
812
|
if (!kb)
|
|
614
813
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
@@ -621,7 +820,7 @@ export class KnowledgeEngine {
|
|
|
621
820
|
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
622
821
|
return activeUpdate;
|
|
623
822
|
}
|
|
624
|
-
const updateRun = this.runUpdate(updateableKB, onProgress, signal).finally(() => {
|
|
823
|
+
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
|
|
625
824
|
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
626
825
|
this.activeUpdates.delete(updateableKB.id);
|
|
627
826
|
});
|
|
@@ -631,16 +830,21 @@ export class KnowledgeEngine {
|
|
|
631
830
|
async runUpdate(kb, onProgress, signal) {
|
|
632
831
|
if (!this.db)
|
|
633
832
|
throw new Error("Engine not initialized");
|
|
833
|
+
throwIfAborted(signal);
|
|
634
834
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
635
835
|
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
636
836
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
637
837
|
let replacementVectorPath;
|
|
638
838
|
let addedVectorPath;
|
|
639
839
|
let addedVectorWriter;
|
|
840
|
+
const insertedChunkIds = [];
|
|
841
|
+
const stagedSymbols = [];
|
|
640
842
|
try {
|
|
641
|
-
const vectorPath =
|
|
843
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
642
844
|
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
643
845
|
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
846
|
+
let latestSkippedTotal = 0;
|
|
847
|
+
let latestSkippedSummary = "none";
|
|
644
848
|
const existingHashes = new Map();
|
|
645
849
|
let existingIndex = 0;
|
|
646
850
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
@@ -687,7 +891,7 @@ export class KnowledgeEngine {
|
|
|
687
891
|
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
688
892
|
}
|
|
689
893
|
addedVectorCount += newVectors.length;
|
|
690
|
-
insertChunks(this.db, kb.id, batch);
|
|
894
|
+
insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
|
|
691
895
|
addedCount += batch.length;
|
|
692
896
|
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
693
897
|
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
@@ -728,10 +932,12 @@ export class KnowledgeEngine {
|
|
|
728
932
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
729
933
|
onProgress?.(message);
|
|
730
934
|
scannedFiles = 1;
|
|
731
|
-
|
|
935
|
+
const chunks = await chunkUrl(kb.source_path, signal);
|
|
936
|
+
stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
|
|
937
|
+
await processChunks(chunks);
|
|
732
938
|
}
|
|
733
939
|
else if (statSync(kb.source_path).isDirectory()) {
|
|
734
|
-
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
940
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
|
|
735
941
|
plannedTotalFiles = plan.files;
|
|
736
942
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
737
943
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -742,11 +948,15 @@ export class KnowledgeEngine {
|
|
|
742
948
|
});
|
|
743
949
|
onProgress?.(planningMessage);
|
|
744
950
|
const skipped = createSkippedScanStats();
|
|
745
|
-
for (const file of
|
|
951
|
+
for (const file of iterateScannableFiles(kb.source_path, skipped, scanOptions)) {
|
|
746
952
|
if (signal?.aborted)
|
|
747
953
|
throw new Error("Cancelled");
|
|
954
|
+
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
955
|
+
if (!extracted)
|
|
956
|
+
continue;
|
|
748
957
|
scannedFiles++;
|
|
749
|
-
|
|
958
|
+
stagedSymbols.push(...extractSymbols(extracted.content, file.relPath, extracted.fileType));
|
|
959
|
+
await processChunks(await chunkFile(extracted.content, file.relPath));
|
|
750
960
|
if (scannedFiles % 25 === 0) {
|
|
751
961
|
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
752
962
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -762,7 +972,9 @@ export class KnowledgeEngine {
|
|
|
762
972
|
onProgress?.(message);
|
|
763
973
|
}
|
|
764
974
|
}
|
|
765
|
-
|
|
975
|
+
latestSkippedTotal = skipped.total;
|
|
976
|
+
latestSkippedSummary = summarizeSkippedScan(skipped);
|
|
977
|
+
const message = `Scanned ${scannedFiles} files, skipped ${skipped.total} (${latestSkippedSummary}), reconciling deletes...`;
|
|
766
978
|
updateIndexingJob(this.db, kb.id, {
|
|
767
979
|
phase: "reconciling",
|
|
768
980
|
message,
|
|
@@ -776,9 +988,10 @@ export class KnowledgeEngine {
|
|
|
776
988
|
onProgress?.(message);
|
|
777
989
|
}
|
|
778
990
|
else {
|
|
779
|
-
const
|
|
991
|
+
const extracted = await extractSourceFileContent(kb.source_path, signal);
|
|
780
992
|
scannedFiles = 1;
|
|
781
|
-
|
|
993
|
+
stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
|
|
994
|
+
await processChunks(await chunkFile(extracted.content, kb.source_path));
|
|
782
995
|
}
|
|
783
996
|
await flushPending();
|
|
784
997
|
addedVectorWriter.close();
|
|
@@ -787,8 +1000,7 @@ export class KnowledgeEngine {
|
|
|
787
1000
|
for (const entries of existingHashes.values()) {
|
|
788
1001
|
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
789
1002
|
}
|
|
790
|
-
|
|
791
|
-
deleteChunksByIds(this.db, idsToRemove);
|
|
1003
|
+
const idsToRemoveSet = new Set(idsToRemove);
|
|
792
1004
|
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
793
1005
|
updateIndexingJob(this.db, kb.id, {
|
|
794
1006
|
phase: "reconciling",
|
|
@@ -811,6 +1023,10 @@ export class KnowledgeEngine {
|
|
|
811
1023
|
};
|
|
812
1024
|
try {
|
|
813
1025
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1026
|
+
if (signal?.aborted)
|
|
1027
|
+
throw new Error("Cancelled");
|
|
1028
|
+
if (idsToRemoveSet.has(chunk.id))
|
|
1029
|
+
continue;
|
|
814
1030
|
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
815
1031
|
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
816
1032
|
const vector = oldVectorReader && oldVectorIndex !== undefined
|
|
@@ -847,9 +1063,14 @@ export class KnowledgeEngine {
|
|
|
847
1063
|
if (addedVectorPath)
|
|
848
1064
|
rmSync(addedVectorPath, { force: true });
|
|
849
1065
|
addedVectorPath = undefined;
|
|
1066
|
+
if (idsToRemove.length > 0)
|
|
1067
|
+
deleteChunksByIds(this.db, idsToRemove);
|
|
1068
|
+
deleteSymbolsByKB(this.db, kb.id);
|
|
1069
|
+
insertSymbols(this.db, kb.id, stagedSymbols);
|
|
850
1070
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
851
1071
|
updateKBStatus(this.db, kb.id, "ready");
|
|
852
|
-
const
|
|
1072
|
+
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
1073
|
+
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
|
|
853
1074
|
updateIndexingJob(this.db, kb.id, {
|
|
854
1075
|
phase: "ready",
|
|
855
1076
|
message: readyMessage,
|
|
@@ -869,14 +1090,19 @@ export class KnowledgeEngine {
|
|
|
869
1090
|
rmSync(replacementVectorPath, { force: true });
|
|
870
1091
|
if (addedVectorPath)
|
|
871
1092
|
rmSync(addedVectorPath, { force: true });
|
|
872
|
-
|
|
1093
|
+
if (insertedChunkIds.length > 0) {
|
|
1094
|
+
deleteChunksByIds(this.db, insertedChunkIds);
|
|
1095
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
1096
|
+
}
|
|
1097
|
+
updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
|
|
873
1098
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
|
|
874
1099
|
throw e;
|
|
875
1100
|
}
|
|
876
1101
|
}
|
|
877
|
-
async search(query, options = {}) {
|
|
1102
|
+
async search(query, options = {}, signal) {
|
|
878
1103
|
if (!this.db)
|
|
879
1104
|
throw new Error("Engine not initialized");
|
|
1105
|
+
throwIfAborted(signal);
|
|
880
1106
|
const requestedMode = options.mode ?? "hybrid";
|
|
881
1107
|
if (requestedMode === "auto") {
|
|
882
1108
|
const primaryMode = chooseAutoMode(query);
|
|
@@ -887,7 +1113,8 @@ export class KnowledgeEngine {
|
|
|
887
1113
|
if (tried.includes(mode))
|
|
888
1114
|
continue;
|
|
889
1115
|
tried.push(mode);
|
|
890
|
-
|
|
1116
|
+
throwIfAborted(signal);
|
|
1117
|
+
const response = await this.search(query, { ...options, mode }, signal);
|
|
891
1118
|
if (response.warnings)
|
|
892
1119
|
warnings.push(...response.warnings);
|
|
893
1120
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
@@ -916,7 +1143,8 @@ export class KnowledgeEngine {
|
|
|
916
1143
|
}
|
|
917
1144
|
const db = this.db;
|
|
918
1145
|
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
919
|
-
const
|
|
1146
|
+
const resolvedMode = retrievalModeFor(mode);
|
|
1147
|
+
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
920
1148
|
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
921
1149
|
const normalizedQuery = normalizedQueryText(query);
|
|
922
1150
|
const queryTokens = tokenizeForSimilarity(query);
|
|
@@ -925,6 +1153,7 @@ export class KnowledgeEngine {
|
|
|
925
1153
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
926
1154
|
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
|
|
927
1155
|
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1156
|
+
const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
|
|
928
1157
|
for (const kb of availableKBs) {
|
|
929
1158
|
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
930
1159
|
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
@@ -942,19 +1171,24 @@ export class KnowledgeEngine {
|
|
|
942
1171
|
const allResults = [];
|
|
943
1172
|
const vectorsByChunkId = new Map();
|
|
944
1173
|
for (const kb of kbs) {
|
|
1174
|
+
throwIfAborted(signal);
|
|
945
1175
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
946
1176
|
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
947
1177
|
}
|
|
948
1178
|
if (kb.chunk_count === 0)
|
|
949
1179
|
continue;
|
|
950
|
-
const vectorPath =
|
|
1180
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
951
1181
|
if (retrievalMode === "fast") {
|
|
952
|
-
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }))
|
|
1182
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
|
|
1183
|
+
...result,
|
|
1184
|
+
score: result.score * kbTrustMultiplier(kb),
|
|
1185
|
+
})));
|
|
953
1186
|
}
|
|
954
1187
|
else if (retrievalMode === "semantic") {
|
|
955
|
-
const queryVec = await embedQuery(query);
|
|
956
|
-
|
|
957
|
-
|
|
1188
|
+
const queryVec = await embedQuery(query, signal);
|
|
1189
|
+
throwIfAborted(signal);
|
|
1190
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
1191
|
+
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
958
1192
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
959
1193
|
vectorsByChunkId.set(chunkId, vector);
|
|
960
1194
|
}
|
|
@@ -964,13 +1198,14 @@ export class KnowledgeEngine {
|
|
|
964
1198
|
if (bm25Results.length === 0)
|
|
965
1199
|
continue;
|
|
966
1200
|
let vecResults = [];
|
|
967
|
-
const queryVec = await embedQuery(query);
|
|
968
|
-
|
|
1201
|
+
const queryVec = await embedQuery(query, signal);
|
|
1202
|
+
throwIfAborted(signal);
|
|
1203
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
969
1204
|
vecResults = vectorResults.results;
|
|
970
1205
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
971
1206
|
vectorsByChunkId.set(chunkId, vector);
|
|
972
1207
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
973
|
-
allResults.push(...fused);
|
|
1208
|
+
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
974
1209
|
}
|
|
975
1210
|
}
|
|
976
1211
|
// Deduplicate and sort
|
|
@@ -983,6 +1218,7 @@ export class KnowledgeEngine {
|
|
|
983
1218
|
});
|
|
984
1219
|
const rankingByChunkId = new Map();
|
|
985
1220
|
for (const result of unique) {
|
|
1221
|
+
throwIfAborted(signal);
|
|
986
1222
|
const chunk = getChunkById(db, result.chunkId);
|
|
987
1223
|
if (chunk) {
|
|
988
1224
|
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
@@ -1030,9 +1266,11 @@ export class KnowledgeEngine {
|
|
|
1030
1266
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
1031
1267
|
})
|
|
1032
1268
|
.filter(Boolean);
|
|
1033
|
-
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
1269
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
|
|
1270
|
+
throwIfAborted(signal);
|
|
1034
1271
|
const ranked = [];
|
|
1035
1272
|
for (const r of reranked) {
|
|
1273
|
+
throwIfAborted(signal);
|
|
1036
1274
|
const chunk = getChunkById(db, r.chunkId);
|
|
1037
1275
|
if (!chunk)
|
|
1038
1276
|
continue;
|
|
@@ -1051,17 +1289,30 @@ export class KnowledgeEngine {
|
|
|
1051
1289
|
}
|
|
1052
1290
|
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1053
1291
|
const page = diversified.slice(offset, offset + limit);
|
|
1054
|
-
const results = page.map((r) =>
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1292
|
+
const results = page.map((r) => {
|
|
1293
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1294
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1295
|
+
return {
|
|
1296
|
+
content: r.content,
|
|
1297
|
+
file_path: r.chunk.file_path,
|
|
1298
|
+
file_type: r.chunk.file_type,
|
|
1299
|
+
kb_name: r.kbName,
|
|
1300
|
+
score: r.score,
|
|
1301
|
+
ranking: r.ranking,
|
|
1302
|
+
snippet: r.snippet,
|
|
1303
|
+
start_line: r.startLine,
|
|
1304
|
+
end_line: r.endLine,
|
|
1305
|
+
provenance: {
|
|
1306
|
+
chunk_id: r.chunk.id,
|
|
1307
|
+
chunk_hash: r.chunk.content_hash,
|
|
1308
|
+
indexed_at: r.chunk.indexed_at,
|
|
1309
|
+
source_mtime: sourceMtime,
|
|
1310
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1311
|
+
match_reason: matchReasonFor(mode),
|
|
1312
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1313
|
+
},
|
|
1314
|
+
};
|
|
1315
|
+
});
|
|
1065
1316
|
return {
|
|
1066
1317
|
results,
|
|
1067
1318
|
total_count: diversified.length,
|
|
@@ -1072,6 +1323,7 @@ export class KnowledgeEngine {
|
|
|
1072
1323
|
}
|
|
1073
1324
|
const ranked = [];
|
|
1074
1325
|
for (const r of filtered) {
|
|
1326
|
+
throwIfAborted(signal);
|
|
1075
1327
|
const chunk = getChunkById(db, r.chunkId);
|
|
1076
1328
|
if (!chunk)
|
|
1077
1329
|
continue;
|
|
@@ -1118,6 +1370,19 @@ export class KnowledgeEngine {
|
|
|
1118
1370
|
snippet: r.snippet,
|
|
1119
1371
|
start_line: r.startLine,
|
|
1120
1372
|
end_line: r.endLine,
|
|
1373
|
+
provenance: (() => {
|
|
1374
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1375
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1376
|
+
return {
|
|
1377
|
+
chunk_id: r.chunk.id,
|
|
1378
|
+
chunk_hash: r.chunk.content_hash,
|
|
1379
|
+
indexed_at: r.chunk.indexed_at,
|
|
1380
|
+
source_mtime: sourceMtime,
|
|
1381
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1382
|
+
match_reason: matchReasonFor(mode),
|
|
1383
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1384
|
+
};
|
|
1385
|
+
})(),
|
|
1121
1386
|
}));
|
|
1122
1387
|
return {
|
|
1123
1388
|
results,
|
|
@@ -1136,31 +1401,94 @@ export class KnowledgeEngine {
|
|
|
1136
1401
|
remove(nameOrId) {
|
|
1137
1402
|
if (!this.db)
|
|
1138
1403
|
return false;
|
|
1404
|
+
if (this.disposing)
|
|
1405
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1406
|
+
if (this.activeMutations.size > 0)
|
|
1407
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1139
1408
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1140
1409
|
if (!kb)
|
|
1141
1410
|
return false;
|
|
1142
1411
|
deleteKB(this.db, kb.id);
|
|
1412
|
+
this.deleteVectorFile(kb.id);
|
|
1143
1413
|
return true;
|
|
1144
1414
|
}
|
|
1145
|
-
list() {
|
|
1415
|
+
list(signal) {
|
|
1146
1416
|
if (!this.db)
|
|
1147
1417
|
return [];
|
|
1418
|
+
throwIfAborted(signal);
|
|
1148
1419
|
return listKBs(this.db);
|
|
1149
1420
|
}
|
|
1150
1421
|
clear() {
|
|
1151
1422
|
if (!this.db)
|
|
1152
1423
|
return;
|
|
1153
|
-
|
|
1424
|
+
if (this.disposing)
|
|
1425
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1426
|
+
if (this.activeMutations.size > 0)
|
|
1427
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1428
|
+
for (const kb of listKBs(this.db)) {
|
|
1154
1429
|
deleteKB(this.db, kb.id);
|
|
1430
|
+
this.deleteVectorFile(kb.id);
|
|
1431
|
+
}
|
|
1155
1432
|
}
|
|
1156
|
-
diagnose() {
|
|
1433
|
+
diagnose(signal) {
|
|
1157
1434
|
if (!this.db)
|
|
1158
1435
|
return [];
|
|
1436
|
+
throwIfAborted(signal);
|
|
1437
|
+
const db = this.db;
|
|
1438
|
+
return listKBs(db).map((kb) => {
|
|
1439
|
+
throwIfAborted(signal);
|
|
1440
|
+
return diagnoseKB(db, kb, signal);
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
symbolSearch(query, options = {}, signal) {
|
|
1444
|
+
if (!this.db)
|
|
1445
|
+
throw new Error("Engine not initialized");
|
|
1446
|
+
throwIfAborted(signal);
|
|
1159
1447
|
const db = this.db;
|
|
1160
|
-
|
|
1448
|
+
const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
|
|
1449
|
+
if (options.kb_id && !selectedKB)
|
|
1450
|
+
throw new Error(`Knowledge base not found: ${options.kb_id}`);
|
|
1451
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1452
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
1453
|
+
const symbolOptions = {
|
|
1454
|
+
kbId: selectedKB?.id,
|
|
1455
|
+
kind: options.kind,
|
|
1456
|
+
filePattern: options.file_pattern,
|
|
1457
|
+
exact: options.exact,
|
|
1458
|
+
};
|
|
1459
|
+
const symbols = searchSymbols(db, query, {
|
|
1460
|
+
...symbolOptions,
|
|
1461
|
+
limit: limit + 1,
|
|
1462
|
+
offset,
|
|
1463
|
+
});
|
|
1464
|
+
throwIfAborted(signal);
|
|
1465
|
+
const page = symbols.slice(0, limit);
|
|
1466
|
+
const total = countSymbols(db, query, symbolOptions);
|
|
1467
|
+
throwIfAborted(signal);
|
|
1468
|
+
return {
|
|
1469
|
+
results: page.map((symbol) => {
|
|
1470
|
+
const kb = getKB(db, symbol.kb_id);
|
|
1471
|
+
return {
|
|
1472
|
+
name: symbol.name,
|
|
1473
|
+
kind: symbol.kind,
|
|
1474
|
+
file_path: symbol.file_path,
|
|
1475
|
+
file_type: symbol.file_type,
|
|
1476
|
+
kb_name: kb?.name ?? "unknown",
|
|
1477
|
+
start_line: symbol.start_line,
|
|
1478
|
+
end_line: symbol.end_line,
|
|
1479
|
+
signature: symbol.signature ?? undefined,
|
|
1480
|
+
container_name: symbol.container_name ?? undefined,
|
|
1481
|
+
text: symbol.text,
|
|
1482
|
+
indexed_at: symbol.indexed_at,
|
|
1483
|
+
};
|
|
1484
|
+
}),
|
|
1485
|
+
total_count: total,
|
|
1486
|
+
has_more: symbols.length > limit,
|
|
1487
|
+
};
|
|
1161
1488
|
}
|
|
1162
|
-
doctor() {
|
|
1163
|
-
|
|
1489
|
+
doctor(signal) {
|
|
1490
|
+
throwIfAborted(signal);
|
|
1491
|
+
const diagnostics = this.diagnose(signal);
|
|
1164
1492
|
const issues = [];
|
|
1165
1493
|
const kbs = this.list();
|
|
1166
1494
|
if (kbs.length === 0) {
|
|
@@ -1168,9 +1496,11 @@ export class KnowledgeEngine {
|
|
|
1168
1496
|
severity: "blocking",
|
|
1169
1497
|
message: "No knowledge bases are indexed.",
|
|
1170
1498
|
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1499
|
+
action_code: "rebuild_kb",
|
|
1171
1500
|
});
|
|
1172
1501
|
}
|
|
1173
1502
|
for (const diagnostic of diagnostics) {
|
|
1503
|
+
throwIfAborted(signal);
|
|
1174
1504
|
if (diagnostic.stuck_indexing) {
|
|
1175
1505
|
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1176
1506
|
issues.push({
|
|
@@ -1178,6 +1508,7 @@ export class KnowledgeEngine {
|
|
|
1178
1508
|
kb_name: diagnostic.kb_name,
|
|
1179
1509
|
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1180
1510
|
action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1511
|
+
action_code: "wait_for_indexing",
|
|
1181
1512
|
});
|
|
1182
1513
|
}
|
|
1183
1514
|
if (diagnostic.status === "error") {
|
|
@@ -1186,6 +1517,7 @@ export class KnowledgeEngine {
|
|
|
1186
1517
|
kb_name: diagnostic.kb_name,
|
|
1187
1518
|
message: "Knowledge base is in error state and is skipped by search.",
|
|
1188
1519
|
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1520
|
+
action_code: "rebuild_kb",
|
|
1189
1521
|
});
|
|
1190
1522
|
}
|
|
1191
1523
|
if (diagnostic.coverage_percent < 80) {
|
|
@@ -1194,6 +1526,7 @@ export class KnowledgeEngine {
|
|
|
1194
1526
|
kb_name: diagnostic.kb_name,
|
|
1195
1527
|
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1196
1528
|
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1529
|
+
action_code: "review_skipped_scope",
|
|
1197
1530
|
});
|
|
1198
1531
|
}
|
|
1199
1532
|
if (diagnostic.stale_files.length > 0) {
|
|
@@ -1202,6 +1535,7 @@ export class KnowledgeEngine {
|
|
|
1202
1535
|
kb_name: diagnostic.kb_name,
|
|
1203
1536
|
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1204
1537
|
action: "Run knowledge_update for this KB.",
|
|
1538
|
+
action_code: "run_update",
|
|
1205
1539
|
});
|
|
1206
1540
|
}
|
|
1207
1541
|
if (diagnostic.orphan_files.length > 0) {
|
|
@@ -1210,6 +1544,7 @@ export class KnowledgeEngine {
|
|
|
1210
1544
|
kb_name: diagnostic.kb_name,
|
|
1211
1545
|
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1212
1546
|
action: "Run knowledge_update or rebuild the KB.",
|
|
1547
|
+
action_code: "run_update",
|
|
1213
1548
|
});
|
|
1214
1549
|
}
|
|
1215
1550
|
if (diagnostic.skipped_files.total > 0) {
|
|
@@ -1221,9 +1556,24 @@ export class KnowledgeEngine {
|
|
|
1221
1556
|
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1222
1557
|
.join(", ")}).`,
|
|
1223
1558
|
action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1559
|
+
action_code: "review_skipped_scope",
|
|
1224
1560
|
});
|
|
1225
1561
|
}
|
|
1226
1562
|
}
|
|
1563
|
+
if (this.db) {
|
|
1564
|
+
for (const kb of kbs) {
|
|
1565
|
+
throwIfAborted(signal);
|
|
1566
|
+
if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
|
|
1567
|
+
issues.push({
|
|
1568
|
+
severity: "info",
|
|
1569
|
+
kb_name: kb.name,
|
|
1570
|
+
message: "No lightweight symbols are indexed for this KB.",
|
|
1571
|
+
action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
|
|
1572
|
+
action_code: "run_update",
|
|
1573
|
+
});
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1227
1577
|
const penalty = issues.reduce((score, issue) => {
|
|
1228
1578
|
if (issue.severity === "blocking")
|
|
1229
1579
|
return score + 35;
|
|
@@ -1242,138 +1592,193 @@ export class KnowledgeEngine {
|
|
|
1242
1592
|
summary,
|
|
1243
1593
|
issues,
|
|
1244
1594
|
diagnostics,
|
|
1595
|
+
actions: issues.map((issue) => ({
|
|
1596
|
+
code: issue.action_code ?? "none",
|
|
1597
|
+
kb_name: issue.kb_name,
|
|
1598
|
+
target: issue.kb_name,
|
|
1599
|
+
description: issue.action,
|
|
1600
|
+
})),
|
|
1245
1601
|
};
|
|
1246
1602
|
}
|
|
1247
|
-
async exportKB(nameOrId, outputPath) {
|
|
1603
|
+
async exportKB(nameOrId, outputPath, signal, onProgress) {
|
|
1248
1604
|
if (!this.db)
|
|
1249
1605
|
throw new Error("Engine not initialized");
|
|
1606
|
+
throwIfAborted(signal);
|
|
1250
1607
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1251
1608
|
if (!kb)
|
|
1252
1609
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
1253
|
-
|
|
1254
|
-
|
|
1610
|
+
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
|
|
1611
|
+
}
|
|
1612
|
+
async exportKBUnlocked(kb, outputPath, signal, onProgress) {
|
|
1613
|
+
if (!this.db)
|
|
1614
|
+
throw new Error("Engine not initialized");
|
|
1615
|
+
const { createWriteStream } = await import("node:fs");
|
|
1616
|
+
const tempOutputPath = tempVectorPath(outputPath);
|
|
1617
|
+
const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
|
|
1618
|
+
let count = 0;
|
|
1255
1619
|
const header = JSON.stringify({
|
|
1256
1620
|
name: kb.name,
|
|
1257
1621
|
description: kb.description,
|
|
1258
1622
|
source_type: "text",
|
|
1259
|
-
chunk_count:
|
|
1623
|
+
chunk_count: kb.chunk_count,
|
|
1260
1624
|
});
|
|
1261
|
-
|
|
1262
|
-
header
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1625
|
+
try {
|
|
1626
|
+
await writeLine(stream, header);
|
|
1627
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1628
|
+
throwIfAborted(signal);
|
|
1629
|
+
await writeLine(stream, JSON.stringify({
|
|
1630
|
+
content: chunk.content,
|
|
1631
|
+
file_path: chunk.file_path,
|
|
1632
|
+
file_type: chunk.file_type,
|
|
1633
|
+
start_line: chunk.start_line,
|
|
1634
|
+
end_line: chunk.end_line,
|
|
1635
|
+
metadata_json: chunk.metadata_json,
|
|
1636
|
+
}));
|
|
1637
|
+
count++;
|
|
1638
|
+
if (count % INDEX_EMBED_BATCH_SIZE === 0)
|
|
1639
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
|
|
1640
|
+
}
|
|
1641
|
+
await finishWriteStream(stream);
|
|
1642
|
+
throwIfAborted(signal);
|
|
1643
|
+
renameSync(tempOutputPath, outputPath);
|
|
1644
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
|
|
1645
|
+
return count;
|
|
1646
|
+
}
|
|
1647
|
+
catch (error) {
|
|
1648
|
+
stream.destroy();
|
|
1649
|
+
rmSync(tempOutputPath, { force: true });
|
|
1650
|
+
throw error;
|
|
1651
|
+
}
|
|
1274
1652
|
}
|
|
1275
1653
|
async importKB(inputPath, onProgress, signal) {
|
|
1276
1654
|
if (!this.db)
|
|
1277
1655
|
throw new Error("Engine not initialized");
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1656
|
+
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
|
|
1657
|
+
}
|
|
1658
|
+
async importKBUnlocked(inputPath, onProgress, signal) {
|
|
1659
|
+
if (!this.db)
|
|
1660
|
+
throw new Error("Engine not initialized");
|
|
1661
|
+
throwIfAborted(signal);
|
|
1662
|
+
const { createReadStream } = await import("node:fs");
|
|
1663
|
+
const { createInterface } = await import("node:readline");
|
|
1664
|
+
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1665
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1666
|
+
let header;
|
|
1667
|
+
let kb;
|
|
1286
1668
|
let vectorWriter;
|
|
1287
1669
|
let tempVectorFile;
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1670
|
+
let inserted = 0;
|
|
1671
|
+
let lineNumber = 0;
|
|
1672
|
+
const pendingChunks = [];
|
|
1673
|
+
const pendingSymbols = [];
|
|
1674
|
+
const flushPending = async () => {
|
|
1675
|
+
if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
|
|
1676
|
+
return;
|
|
1677
|
+
throwIfAborted(signal);
|
|
1678
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
1679
|
+
const symbols = pendingSymbols.splice(0);
|
|
1680
|
+
const total = header?.chunk_count;
|
|
1681
|
+
const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
|
|
1291
1682
|
updateIndexingJob(this.db, kb.id, {
|
|
1292
|
-
phase: "
|
|
1293
|
-
message
|
|
1294
|
-
|
|
1683
|
+
phase: "embedding",
|
|
1684
|
+
message,
|
|
1685
|
+
processed_chunks: inserted,
|
|
1686
|
+
total_files: total,
|
|
1687
|
+
added_chunks: inserted,
|
|
1295
1688
|
});
|
|
1296
|
-
onProgress?.(
|
|
1297
|
-
const
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1689
|
+
onProgress?.(message);
|
|
1690
|
+
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1691
|
+
throwIfAborted(signal);
|
|
1692
|
+
insertChunks(this.db, kb.id, batch);
|
|
1693
|
+
insertSymbols(this.db, kb.id, symbols);
|
|
1694
|
+
vectorWriter.append(vectors);
|
|
1695
|
+
inserted += batch.length;
|
|
1696
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1697
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1698
|
+
phase: "storing",
|
|
1699
|
+
message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
|
|
1700
|
+
processed_chunks: inserted,
|
|
1701
|
+
total_files: total,
|
|
1702
|
+
added_chunks: inserted,
|
|
1703
|
+
});
|
|
1704
|
+
};
|
|
1705
|
+
try {
|
|
1706
|
+
for await (const rawLine of lines) {
|
|
1707
|
+
throwIfAborted(signal);
|
|
1708
|
+
lineNumber++;
|
|
1709
|
+
const line = rawLine.trim();
|
|
1710
|
+
if (!line)
|
|
1711
|
+
continue;
|
|
1712
|
+
if (!header) {
|
|
1713
|
+
header = JSON.parse(line);
|
|
1714
|
+
if (!header.name)
|
|
1715
|
+
throw new Error("Import header is missing a knowledge base name");
|
|
1716
|
+
const existingKB = getKBByName(this.db, header.name);
|
|
1717
|
+
if (existingKB) {
|
|
1718
|
+
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
1719
|
+
}
|
|
1720
|
+
kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
1721
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
1722
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1723
|
+
const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
|
|
1724
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1725
|
+
phase: "importing",
|
|
1726
|
+
message: importMessage,
|
|
1727
|
+
total_files: header.chunk_count,
|
|
1728
|
+
});
|
|
1729
|
+
onProgress?.(importMessage);
|
|
1730
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
1731
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1732
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1733
|
+
continue;
|
|
1734
|
+
}
|
|
1735
|
+
const imported = JSON.parse(line);
|
|
1736
|
+
const chunk = importedChunkToInsert(imported);
|
|
1737
|
+
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
1738
|
+
pendingChunks.push(chunk);
|
|
1739
|
+
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
1740
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
|
|
1741
|
+
await flushPending();
|
|
1349
1742
|
}
|
|
1743
|
+
if (!header)
|
|
1744
|
+
throw new Error("Empty import file");
|
|
1745
|
+
if (!kb || !vectorWriter || !tempVectorFile)
|
|
1746
|
+
throw new Error("Import did not create a knowledge base");
|
|
1747
|
+
await flushPending();
|
|
1350
1748
|
vectorWriter.close();
|
|
1351
1749
|
vectorWriter = undefined;
|
|
1352
|
-
renameSync(tempVectorFile,
|
|
1750
|
+
renameSync(tempVectorFile, this.vectorPathFor(kb.id));
|
|
1353
1751
|
tempVectorFile = undefined;
|
|
1354
|
-
updateKBCounts(this.db, kb.id,
|
|
1752
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1355
1753
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1356
1754
|
const savedKB = getKB(this.db, kb.id);
|
|
1357
1755
|
if (!savedKB)
|
|
1358
1756
|
throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1359
|
-
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${
|
|
1360
|
-
return { kb: savedKB, chunkCount:
|
|
1757
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
|
|
1758
|
+
return { kb: savedKB, chunkCount: inserted };
|
|
1361
1759
|
}
|
|
1362
|
-
catch (
|
|
1760
|
+
catch (error) {
|
|
1761
|
+
lines.close();
|
|
1762
|
+
stream.destroy();
|
|
1363
1763
|
vectorWriter?.close();
|
|
1364
1764
|
if (tempVectorFile)
|
|
1365
1765
|
rmSync(tempVectorFile, { force: true });
|
|
1366
|
-
if (this.db) {
|
|
1367
|
-
finishIndexingJob(this.db, kb.id, isCancellationError(
|
|
1766
|
+
if (this.db && kb) {
|
|
1767
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
|
|
1768
|
+
deleteKB(this.db, kb.id);
|
|
1769
|
+
this.deleteVectorFile(kb.id);
|
|
1368
1770
|
}
|
|
1369
|
-
|
|
1370
|
-
throw e;
|
|
1771
|
+
throw error;
|
|
1371
1772
|
}
|
|
1372
1773
|
}
|
|
1373
1774
|
async dispose(options = {}) {
|
|
1775
|
+
this.disposing = true;
|
|
1374
1776
|
const activeUpdates = [...this.activeUpdates.values()];
|
|
1375
1777
|
if (activeUpdates.length > 0)
|
|
1376
1778
|
await Promise.allSettled(activeUpdates);
|
|
1779
|
+
const activeMutations = [...this.activeMutations.values()];
|
|
1780
|
+
if (activeMutations.length > 0)
|
|
1781
|
+
await Promise.allSettled(activeMutations);
|
|
1377
1782
|
const disposeModels = options.disposeModels ?? true;
|
|
1378
1783
|
if (disposeModels) {
|
|
1379
1784
|
await disposeEmbedding();
|