pi-knowledge 0.4.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +19 -8
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +63 -10
- package/dist/src/engine.js +538 -164
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +213 -8
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +4 -2
package/dist/src/engine.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
4
|
import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
6
|
import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
|
+
import { extractSymbols } from "./indexer/symbols.js";
|
|
7
8
|
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
8
9
|
import { searchBM25 } from "./search/bm25.js";
|
|
9
10
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
|
11
12
|
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
12
13
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
13
14
|
import { searchVectorFile } from "./search/vector.js";
|
|
14
|
-
import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile,
|
|
15
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
|
|
15
16
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
16
17
|
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
17
18
|
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
@@ -24,6 +25,26 @@ function isCancellationError(error) {
|
|
|
24
25
|
function tempVectorPath(vectorPath) {
|
|
25
26
|
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
26
27
|
}
|
|
28
|
+
function retrievalModeFor(mode) {
|
|
29
|
+
if (mode === "auto")
|
|
30
|
+
return "hybrid";
|
|
31
|
+
if (mode === "code" || mode === "config" || mode === "errors")
|
|
32
|
+
return "fast";
|
|
33
|
+
if (mode === "docs" || mode === "decision")
|
|
34
|
+
return "hybrid";
|
|
35
|
+
return mode;
|
|
36
|
+
}
|
|
37
|
+
function matchReasonFor(mode) {
|
|
38
|
+
if (mode === "deep")
|
|
39
|
+
return "rerank";
|
|
40
|
+
if (mode === "adaptive")
|
|
41
|
+
return "adaptive";
|
|
42
|
+
if (mode === "semantic")
|
|
43
|
+
return "vector";
|
|
44
|
+
if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
|
|
45
|
+
return "bm25";
|
|
46
|
+
return "hybrid";
|
|
47
|
+
}
|
|
27
48
|
function toScanOptions(options = {}) {
|
|
28
49
|
return {
|
|
29
50
|
includeSuggestedText: options.include_suggested_text === true,
|
|
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
|
|
|
60
81
|
return {};
|
|
61
82
|
}
|
|
62
83
|
}
|
|
63
|
-
function planDirectoryScan(dirPath, options = {}) {
|
|
84
|
+
function planDirectoryScan(dirPath, options = {}, signal) {
|
|
64
85
|
const skipped = createSkippedScanStats();
|
|
65
86
|
let files = 0;
|
|
66
87
|
let bytes = 0;
|
|
67
88
|
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
89
|
+
throwIfAborted(signal);
|
|
68
90
|
files++;
|
|
69
91
|
bytes += file.size;
|
|
70
92
|
}
|
|
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
|
|
|
76
98
|
skipped,
|
|
77
99
|
};
|
|
78
100
|
}
|
|
101
|
+
function sourceMtimeFor(kb, filePath) {
|
|
102
|
+
if (!kb?.source_path)
|
|
103
|
+
return undefined;
|
|
104
|
+
const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
|
|
105
|
+
try {
|
|
106
|
+
return statSync(absPath).mtimeMs;
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function kbTrustMultiplier(kb) {
|
|
113
|
+
let multiplier = kb.status === "stale" ? 0.92 : 1;
|
|
114
|
+
if (kb.source_type === "directory" || kb.source_type === "file")
|
|
115
|
+
multiplier *= 1.04;
|
|
116
|
+
if (kb.source_type === "text" || kb.source_type === "url")
|
|
117
|
+
multiplier *= 0.98;
|
|
118
|
+
return multiplier;
|
|
119
|
+
}
|
|
79
120
|
function formatBytes(bytes) {
|
|
80
121
|
if (bytes < 1024)
|
|
81
122
|
return `${bytes} B`;
|
|
@@ -347,21 +388,143 @@ async function chunkUrl(source, signal) {
|
|
|
347
388
|
function normalizeExtractedText(text) {
|
|
348
389
|
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
349
390
|
}
|
|
391
|
+
async function extractSourceFileContent(filePath, signal) {
|
|
392
|
+
const lowerPath = filePath.toLowerCase();
|
|
393
|
+
if (lowerPath.endsWith(".pdf")) {
|
|
394
|
+
throwIfAborted(signal);
|
|
395
|
+
// PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
396
|
+
const { extractText } = await import("unpdf");
|
|
397
|
+
throwIfAborted(signal);
|
|
398
|
+
const buf = readFileSync(filePath);
|
|
399
|
+
throwIfAborted(signal);
|
|
400
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
401
|
+
throwIfAborted(signal);
|
|
402
|
+
return { content: normalizeExtractedText(text), fileType: "pdf" };
|
|
403
|
+
}
|
|
404
|
+
if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
|
|
405
|
+
throwIfAborted(signal);
|
|
406
|
+
// DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
407
|
+
const mammoth = await import("mammoth");
|
|
408
|
+
throwIfAborted(signal);
|
|
409
|
+
const result = await mammoth.extractRawText({ path: filePath });
|
|
410
|
+
throwIfAborted(signal);
|
|
411
|
+
return { content: result.value, fileType: "docx" };
|
|
412
|
+
}
|
|
413
|
+
if (!isReadableTextFile(filePath)) {
|
|
414
|
+
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
415
|
+
}
|
|
416
|
+
throwIfAborted(signal);
|
|
417
|
+
return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
|
|
418
|
+
}
|
|
419
|
+
function throwIfAborted(signal) {
|
|
420
|
+
if (signal?.aborted)
|
|
421
|
+
throw new Error("Cancelled");
|
|
422
|
+
}
|
|
423
|
+
function importedChunkToInsert(chunk) {
|
|
424
|
+
const metadataJson = chunk.metadata_json || "{}";
|
|
425
|
+
return {
|
|
426
|
+
content_hash: chunkIdentityHash({
|
|
427
|
+
content: chunk.content,
|
|
428
|
+
filePath: chunk.file_path,
|
|
429
|
+
fileType: chunk.file_type,
|
|
430
|
+
startLine: chunk.start_line,
|
|
431
|
+
endLine: chunk.end_line,
|
|
432
|
+
metadataJson,
|
|
433
|
+
}),
|
|
434
|
+
content: chunk.content,
|
|
435
|
+
content_tokenized: "",
|
|
436
|
+
file_path: chunk.file_path,
|
|
437
|
+
file_type: chunk.file_type,
|
|
438
|
+
start_line: chunk.start_line,
|
|
439
|
+
end_line: chunk.end_line,
|
|
440
|
+
metadata_json: metadataJson,
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
async function writeLine(stream, line) {
|
|
444
|
+
if (stream.write(`${line}\n`))
|
|
445
|
+
return;
|
|
446
|
+
await new Promise((resolve, reject) => {
|
|
447
|
+
const cleanup = () => {
|
|
448
|
+
stream.off("drain", onDrain);
|
|
449
|
+
stream.off("error", onError);
|
|
450
|
+
};
|
|
451
|
+
const onDrain = () => {
|
|
452
|
+
cleanup();
|
|
453
|
+
resolve();
|
|
454
|
+
};
|
|
455
|
+
const onError = (error) => {
|
|
456
|
+
cleanup();
|
|
457
|
+
reject(error);
|
|
458
|
+
};
|
|
459
|
+
stream.once("drain", onDrain);
|
|
460
|
+
stream.once("error", onError);
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
async function finishWriteStream(stream) {
|
|
464
|
+
await new Promise((resolve, reject) => {
|
|
465
|
+
const cleanup = () => {
|
|
466
|
+
stream.off("finish", onFinish);
|
|
467
|
+
stream.off("error", onError);
|
|
468
|
+
};
|
|
469
|
+
const onFinish = () => {
|
|
470
|
+
cleanup();
|
|
471
|
+
resolve();
|
|
472
|
+
};
|
|
473
|
+
const onError = (error) => {
|
|
474
|
+
cleanup();
|
|
475
|
+
reject(error);
|
|
476
|
+
};
|
|
477
|
+
stream.once("finish", onFinish);
|
|
478
|
+
stream.once("error", onError);
|
|
479
|
+
stream.end();
|
|
480
|
+
});
|
|
481
|
+
}
|
|
350
482
|
export class KnowledgeEngine {
|
|
351
483
|
db = null;
|
|
352
484
|
knowledgeDir = "";
|
|
485
|
+
activeUpdates = new Map();
|
|
486
|
+
activeMutations = new Map();
|
|
487
|
+
disposing = false;
|
|
488
|
+
runExclusive(keys, description, operation) {
|
|
489
|
+
if (this.disposing)
|
|
490
|
+
throw new Error("Knowledge engine is shutting down");
|
|
491
|
+
const mutationKeys = Array.isArray(keys) ? keys : [keys];
|
|
492
|
+
for (const key of mutationKeys) {
|
|
493
|
+
if (this.activeMutations.has(key))
|
|
494
|
+
throw new Error(`${description} is already running`);
|
|
495
|
+
}
|
|
496
|
+
const run = Promise.resolve()
|
|
497
|
+
.then(operation)
|
|
498
|
+
.finally(() => {
|
|
499
|
+
for (const key of mutationKeys) {
|
|
500
|
+
if (this.activeMutations.get(key) === run)
|
|
501
|
+
this.activeMutations.delete(key);
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
for (const key of mutationKeys)
|
|
505
|
+
this.activeMutations.set(key, run);
|
|
506
|
+
return run;
|
|
507
|
+
}
|
|
353
508
|
async initialize(knowledgeDir) {
|
|
354
509
|
this.knowledgeDir = knowledgeDir;
|
|
355
510
|
this.db = openDatabase(knowledgeDir);
|
|
511
|
+
this.disposing = false;
|
|
512
|
+
}
|
|
513
|
+
vectorPathFor(kbId) {
|
|
514
|
+
return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
515
|
+
}
|
|
516
|
+
deleteVectorFile(kbId) {
|
|
517
|
+
rmSync(this.vectorPathFor(kbId), { force: true });
|
|
356
518
|
}
|
|
357
|
-
plan(source, options = {}) {
|
|
519
|
+
plan(source, options = {}, signal) {
|
|
520
|
+
throwIfAborted(signal);
|
|
358
521
|
const resolvedSource = resolve(source);
|
|
359
522
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
360
523
|
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
361
524
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
362
525
|
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
363
526
|
if (isDir) {
|
|
364
|
-
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
527
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
|
|
365
528
|
return {
|
|
366
529
|
source_type: "directory",
|
|
367
530
|
scannable_files: plan.files,
|
|
@@ -404,8 +567,13 @@ export class KnowledgeEngine {
|
|
|
404
567
|
};
|
|
405
568
|
}
|
|
406
569
|
async add(source, name, onProgress, signal, options = {}) {
|
|
570
|
+
throwIfAborted(signal);
|
|
571
|
+
return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
|
|
572
|
+
}
|
|
573
|
+
async addUnlocked(source, name, onProgress, signal, options = {}) {
|
|
407
574
|
if (!this.db)
|
|
408
575
|
throw new Error("Engine not initialized");
|
|
576
|
+
throwIfAborted(signal);
|
|
409
577
|
const db = this.db;
|
|
410
578
|
const resolvedSource = resolve(source);
|
|
411
579
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
@@ -428,7 +596,7 @@ export class KnowledgeEngine {
|
|
|
428
596
|
let vectorWriter;
|
|
429
597
|
let tempVectorFile;
|
|
430
598
|
try {
|
|
431
|
-
const vectorPath =
|
|
599
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
432
600
|
tempVectorFile = tempVectorPath(vectorPath);
|
|
433
601
|
const writer = openVectorWriter(tempVectorFile);
|
|
434
602
|
vectorWriter = writer;
|
|
@@ -487,33 +655,26 @@ export class KnowledgeEngine {
|
|
|
487
655
|
await flushPending(processedFiles, totalFiles);
|
|
488
656
|
}
|
|
489
657
|
};
|
|
658
|
+
const addSymbols = (content, filePath, fileType) => {
|
|
659
|
+
insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
|
|
660
|
+
};
|
|
490
661
|
if (isUrl) {
|
|
491
662
|
const message = `Fetching ${source}...`;
|
|
492
663
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
493
664
|
onProgress?.(message);
|
|
494
665
|
const chunks = await chunkUrl(source, signal);
|
|
495
666
|
fileCount = 1;
|
|
667
|
+
addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
|
|
496
668
|
await addChunks(chunks);
|
|
497
669
|
}
|
|
498
|
-
else if (isFile
|
|
499
|
-
|
|
500
|
-
onProgress?.("Extracting text from PDF...");
|
|
501
|
-
const { extractText } = await import("unpdf");
|
|
502
|
-
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
503
|
-
const { text } = await extractText(new Uint8Array(buf));
|
|
504
|
-
fileCount = 1;
|
|
505
|
-
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
506
|
-
}
|
|
507
|
-
else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
508
|
-
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
509
|
-
onProgress?.("Extracting text from DOCX...");
|
|
510
|
-
const mammoth = await import("mammoth");
|
|
511
|
-
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
670
|
+
else if (isFile) {
|
|
671
|
+
const extracted = await extractSourceFileContent(resolvedSource, signal);
|
|
512
672
|
fileCount = 1;
|
|
513
|
-
|
|
673
|
+
addSymbols(extracted.content, resolvedSource, extracted.fileType);
|
|
674
|
+
await addChunks(await chunkFile(extracted.content, resolvedSource));
|
|
514
675
|
}
|
|
515
676
|
else if (isDir) {
|
|
516
|
-
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
677
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
|
|
517
678
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
518
679
|
updateIndexingJob(this.db, kb.id, {
|
|
519
680
|
phase: "planning",
|
|
@@ -540,6 +701,7 @@ export class KnowledgeEngine {
|
|
|
540
701
|
latestSkippedTotal = skipped.total;
|
|
541
702
|
if (chunks.length > 0)
|
|
542
703
|
fileCount++;
|
|
704
|
+
addSymbols(file.content, file.relPath, file.fileType);
|
|
543
705
|
await addChunks(chunks, processedFiles, plan.files);
|
|
544
706
|
if (processedFiles % 25 === 0)
|
|
545
707
|
reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
@@ -556,17 +718,9 @@ export class KnowledgeEngine {
|
|
|
556
718
|
});
|
|
557
719
|
onProgress?.(finalizingMessage);
|
|
558
720
|
}
|
|
559
|
-
else if (isFile) {
|
|
560
|
-
if (!isReadableTextFile(resolvedSource)) {
|
|
561
|
-
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
562
|
-
}
|
|
563
|
-
const { readFileSync } = await import("node:fs");
|
|
564
|
-
const content = readFileSync(resolvedSource, "utf-8");
|
|
565
|
-
fileCount = 1;
|
|
566
|
-
await addChunks(await chunkFile(content, resolvedSource));
|
|
567
|
-
}
|
|
568
721
|
else {
|
|
569
722
|
fileCount = 1;
|
|
723
|
+
addSymbols(source, "inline-text", "text");
|
|
570
724
|
await addChunks(await chunkFile(source, "inline-text"));
|
|
571
725
|
}
|
|
572
726
|
await flushPending();
|
|
@@ -599,29 +753,51 @@ export class KnowledgeEngine {
|
|
|
599
753
|
rmSync(tempVectorFile, { force: true });
|
|
600
754
|
if (this.db) {
|
|
601
755
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
|
|
756
|
+
deleteKB(this.db, kb.id);
|
|
757
|
+
this.deleteVectorFile(kb.id);
|
|
602
758
|
}
|
|
603
|
-
// Clean up partial state on failure
|
|
604
|
-
deleteKB(this.db, kb.id);
|
|
605
759
|
throw e;
|
|
606
760
|
}
|
|
607
761
|
}
|
|
608
762
|
async update(nameOrId, onProgress, signal) {
|
|
609
763
|
if (!this.db)
|
|
610
764
|
throw new Error("Engine not initialized");
|
|
765
|
+
if (this.disposing)
|
|
766
|
+
throw new Error("Knowledge engine is shutting down");
|
|
767
|
+
throwIfAborted(signal);
|
|
611
768
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
612
769
|
if (!kb)
|
|
613
770
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
614
771
|
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
615
772
|
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
616
773
|
}
|
|
774
|
+
const updateableKB = { ...kb, source_path: kb.source_path };
|
|
775
|
+
const activeUpdate = this.activeUpdates.get(updateableKB.id);
|
|
776
|
+
if (activeUpdate) {
|
|
777
|
+
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
778
|
+
return activeUpdate;
|
|
779
|
+
}
|
|
780
|
+
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
|
|
781
|
+
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
782
|
+
this.activeUpdates.delete(updateableKB.id);
|
|
783
|
+
});
|
|
784
|
+
this.activeUpdates.set(updateableKB.id, updateRun);
|
|
785
|
+
return updateRun;
|
|
786
|
+
}
|
|
787
|
+
async runUpdate(kb, onProgress, signal) {
|
|
788
|
+
if (!this.db)
|
|
789
|
+
throw new Error("Engine not initialized");
|
|
790
|
+
throwIfAborted(signal);
|
|
617
791
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
618
792
|
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
619
793
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
620
794
|
let replacementVectorPath;
|
|
621
795
|
let addedVectorPath;
|
|
622
796
|
let addedVectorWriter;
|
|
797
|
+
const insertedChunkIds = [];
|
|
798
|
+
const stagedSymbols = [];
|
|
623
799
|
try {
|
|
624
|
-
const vectorPath =
|
|
800
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
625
801
|
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
626
802
|
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
627
803
|
const existingHashes = new Map();
|
|
@@ -670,7 +846,7 @@ export class KnowledgeEngine {
|
|
|
670
846
|
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
671
847
|
}
|
|
672
848
|
addedVectorCount += newVectors.length;
|
|
673
|
-
insertChunks(this.db, kb.id, batch);
|
|
849
|
+
insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
|
|
674
850
|
addedCount += batch.length;
|
|
675
851
|
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
676
852
|
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
@@ -711,10 +887,12 @@ export class KnowledgeEngine {
|
|
|
711
887
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
712
888
|
onProgress?.(message);
|
|
713
889
|
scannedFiles = 1;
|
|
714
|
-
|
|
890
|
+
const chunks = await chunkUrl(kb.source_path, signal);
|
|
891
|
+
stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
|
|
892
|
+
await processChunks(chunks);
|
|
715
893
|
}
|
|
716
894
|
else if (statSync(kb.source_path).isDirectory()) {
|
|
717
|
-
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
895
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
|
|
718
896
|
plannedTotalFiles = plan.files;
|
|
719
897
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
720
898
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -729,6 +907,7 @@ export class KnowledgeEngine {
|
|
|
729
907
|
if (signal?.aborted)
|
|
730
908
|
throw new Error("Cancelled");
|
|
731
909
|
scannedFiles++;
|
|
910
|
+
stagedSymbols.push(...extractSymbols(file.content, file.relPath, file.fileType));
|
|
732
911
|
await processChunks(await chunkFile(file.content, file.relPath));
|
|
733
912
|
if (scannedFiles % 25 === 0) {
|
|
734
913
|
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
@@ -759,9 +938,10 @@ export class KnowledgeEngine {
|
|
|
759
938
|
onProgress?.(message);
|
|
760
939
|
}
|
|
761
940
|
else {
|
|
762
|
-
const
|
|
941
|
+
const extracted = await extractSourceFileContent(kb.source_path, signal);
|
|
763
942
|
scannedFiles = 1;
|
|
764
|
-
|
|
943
|
+
stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
|
|
944
|
+
await processChunks(await chunkFile(extracted.content, kb.source_path));
|
|
765
945
|
}
|
|
766
946
|
await flushPending();
|
|
767
947
|
addedVectorWriter.close();
|
|
@@ -770,8 +950,7 @@ export class KnowledgeEngine {
|
|
|
770
950
|
for (const entries of existingHashes.values()) {
|
|
771
951
|
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
772
952
|
}
|
|
773
|
-
|
|
774
|
-
deleteChunksByIds(this.db, idsToRemove);
|
|
953
|
+
const idsToRemoveSet = new Set(idsToRemove);
|
|
775
954
|
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
776
955
|
updateIndexingJob(this.db, kb.id, {
|
|
777
956
|
phase: "reconciling",
|
|
@@ -794,6 +973,10 @@ export class KnowledgeEngine {
|
|
|
794
973
|
};
|
|
795
974
|
try {
|
|
796
975
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
976
|
+
if (signal?.aborted)
|
|
977
|
+
throw new Error("Cancelled");
|
|
978
|
+
if (idsToRemoveSet.has(chunk.id))
|
|
979
|
+
continue;
|
|
797
980
|
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
798
981
|
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
799
982
|
const vector = oldVectorReader && oldVectorIndex !== undefined
|
|
@@ -830,6 +1013,10 @@ export class KnowledgeEngine {
|
|
|
830
1013
|
if (addedVectorPath)
|
|
831
1014
|
rmSync(addedVectorPath, { force: true });
|
|
832
1015
|
addedVectorPath = undefined;
|
|
1016
|
+
if (idsToRemove.length > 0)
|
|
1017
|
+
deleteChunksByIds(this.db, idsToRemove);
|
|
1018
|
+
deleteSymbolsByKB(this.db, kb.id);
|
|
1019
|
+
insertSymbols(this.db, kb.id, stagedSymbols);
|
|
833
1020
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
834
1021
|
updateKBStatus(this.db, kb.id, "ready");
|
|
835
1022
|
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
@@ -852,14 +1039,19 @@ export class KnowledgeEngine {
|
|
|
852
1039
|
rmSync(replacementVectorPath, { force: true });
|
|
853
1040
|
if (addedVectorPath)
|
|
854
1041
|
rmSync(addedVectorPath, { force: true });
|
|
855
|
-
|
|
1042
|
+
if (insertedChunkIds.length > 0) {
|
|
1043
|
+
deleteChunksByIds(this.db, insertedChunkIds);
|
|
1044
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
1045
|
+
}
|
|
1046
|
+
updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
|
|
856
1047
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
|
|
857
1048
|
throw e;
|
|
858
1049
|
}
|
|
859
1050
|
}
|
|
860
|
-
async search(query, options = {}) {
|
|
1051
|
+
async search(query, options = {}, signal) {
|
|
861
1052
|
if (!this.db)
|
|
862
1053
|
throw new Error("Engine not initialized");
|
|
1054
|
+
throwIfAborted(signal);
|
|
863
1055
|
const requestedMode = options.mode ?? "hybrid";
|
|
864
1056
|
if (requestedMode === "auto") {
|
|
865
1057
|
const primaryMode = chooseAutoMode(query);
|
|
@@ -870,7 +1062,8 @@ export class KnowledgeEngine {
|
|
|
870
1062
|
if (tried.includes(mode))
|
|
871
1063
|
continue;
|
|
872
1064
|
tried.push(mode);
|
|
873
|
-
|
|
1065
|
+
throwIfAborted(signal);
|
|
1066
|
+
const response = await this.search(query, { ...options, mode }, signal);
|
|
874
1067
|
if (response.warnings)
|
|
875
1068
|
warnings.push(...response.warnings);
|
|
876
1069
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
@@ -899,7 +1092,8 @@ export class KnowledgeEngine {
|
|
|
899
1092
|
}
|
|
900
1093
|
const db = this.db;
|
|
901
1094
|
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
902
|
-
const
|
|
1095
|
+
const resolvedMode = retrievalModeFor(mode);
|
|
1096
|
+
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
903
1097
|
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
904
1098
|
const normalizedQuery = normalizedQueryText(query);
|
|
905
1099
|
const queryTokens = tokenizeForSimilarity(query);
|
|
@@ -908,6 +1102,7 @@ export class KnowledgeEngine {
|
|
|
908
1102
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
909
1103
|
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
|
|
910
1104
|
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1105
|
+
const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
|
|
911
1106
|
for (const kb of availableKBs) {
|
|
912
1107
|
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
913
1108
|
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
@@ -925,19 +1120,24 @@ export class KnowledgeEngine {
|
|
|
925
1120
|
const allResults = [];
|
|
926
1121
|
const vectorsByChunkId = new Map();
|
|
927
1122
|
for (const kb of kbs) {
|
|
1123
|
+
throwIfAborted(signal);
|
|
928
1124
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
929
1125
|
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
930
1126
|
}
|
|
931
1127
|
if (kb.chunk_count === 0)
|
|
932
1128
|
continue;
|
|
933
|
-
const vectorPath =
|
|
1129
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
934
1130
|
if (retrievalMode === "fast") {
|
|
935
|
-
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }))
|
|
1131
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
|
|
1132
|
+
...result,
|
|
1133
|
+
score: result.score * kbTrustMultiplier(kb),
|
|
1134
|
+
})));
|
|
936
1135
|
}
|
|
937
1136
|
else if (retrievalMode === "semantic") {
|
|
938
|
-
const queryVec = await embedQuery(query);
|
|
939
|
-
|
|
940
|
-
|
|
1137
|
+
const queryVec = await embedQuery(query, signal);
|
|
1138
|
+
throwIfAborted(signal);
|
|
1139
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
1140
|
+
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
941
1141
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
942
1142
|
vectorsByChunkId.set(chunkId, vector);
|
|
943
1143
|
}
|
|
@@ -947,13 +1147,14 @@ export class KnowledgeEngine {
|
|
|
947
1147
|
if (bm25Results.length === 0)
|
|
948
1148
|
continue;
|
|
949
1149
|
let vecResults = [];
|
|
950
|
-
const queryVec = await embedQuery(query);
|
|
951
|
-
|
|
1150
|
+
const queryVec = await embedQuery(query, signal);
|
|
1151
|
+
throwIfAborted(signal);
|
|
1152
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
952
1153
|
vecResults = vectorResults.results;
|
|
953
1154
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
954
1155
|
vectorsByChunkId.set(chunkId, vector);
|
|
955
1156
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
956
|
-
allResults.push(...fused);
|
|
1157
|
+
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
957
1158
|
}
|
|
958
1159
|
}
|
|
959
1160
|
// Deduplicate and sort
|
|
@@ -966,6 +1167,7 @@ export class KnowledgeEngine {
|
|
|
966
1167
|
});
|
|
967
1168
|
const rankingByChunkId = new Map();
|
|
968
1169
|
for (const result of unique) {
|
|
1170
|
+
throwIfAborted(signal);
|
|
969
1171
|
const chunk = getChunkById(db, result.chunkId);
|
|
970
1172
|
if (chunk) {
|
|
971
1173
|
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
@@ -1013,9 +1215,11 @@ export class KnowledgeEngine {
|
|
|
1013
1215
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
1014
1216
|
})
|
|
1015
1217
|
.filter(Boolean);
|
|
1016
|
-
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
1218
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
|
|
1219
|
+
throwIfAborted(signal);
|
|
1017
1220
|
const ranked = [];
|
|
1018
1221
|
for (const r of reranked) {
|
|
1222
|
+
throwIfAborted(signal);
|
|
1019
1223
|
const chunk = getChunkById(db, r.chunkId);
|
|
1020
1224
|
if (!chunk)
|
|
1021
1225
|
continue;
|
|
@@ -1034,17 +1238,30 @@ export class KnowledgeEngine {
|
|
|
1034
1238
|
}
|
|
1035
1239
|
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1036
1240
|
const page = diversified.slice(offset, offset + limit);
|
|
1037
|
-
const results = page.map((r) =>
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1241
|
+
const results = page.map((r) => {
|
|
1242
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1243
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1244
|
+
return {
|
|
1245
|
+
content: r.content,
|
|
1246
|
+
file_path: r.chunk.file_path,
|
|
1247
|
+
file_type: r.chunk.file_type,
|
|
1248
|
+
kb_name: r.kbName,
|
|
1249
|
+
score: r.score,
|
|
1250
|
+
ranking: r.ranking,
|
|
1251
|
+
snippet: r.snippet,
|
|
1252
|
+
start_line: r.startLine,
|
|
1253
|
+
end_line: r.endLine,
|
|
1254
|
+
provenance: {
|
|
1255
|
+
chunk_id: r.chunk.id,
|
|
1256
|
+
chunk_hash: r.chunk.content_hash,
|
|
1257
|
+
indexed_at: r.chunk.indexed_at,
|
|
1258
|
+
source_mtime: sourceMtime,
|
|
1259
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1260
|
+
match_reason: matchReasonFor(mode),
|
|
1261
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1262
|
+
},
|
|
1263
|
+
};
|
|
1264
|
+
});
|
|
1048
1265
|
return {
|
|
1049
1266
|
results,
|
|
1050
1267
|
total_count: diversified.length,
|
|
@@ -1055,6 +1272,7 @@ export class KnowledgeEngine {
|
|
|
1055
1272
|
}
|
|
1056
1273
|
const ranked = [];
|
|
1057
1274
|
for (const r of filtered) {
|
|
1275
|
+
throwIfAborted(signal);
|
|
1058
1276
|
const chunk = getChunkById(db, r.chunkId);
|
|
1059
1277
|
if (!chunk)
|
|
1060
1278
|
continue;
|
|
@@ -1101,6 +1319,19 @@ export class KnowledgeEngine {
|
|
|
1101
1319
|
snippet: r.snippet,
|
|
1102
1320
|
start_line: r.startLine,
|
|
1103
1321
|
end_line: r.endLine,
|
|
1322
|
+
provenance: (() => {
|
|
1323
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1324
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1325
|
+
return {
|
|
1326
|
+
chunk_id: r.chunk.id,
|
|
1327
|
+
chunk_hash: r.chunk.content_hash,
|
|
1328
|
+
indexed_at: r.chunk.indexed_at,
|
|
1329
|
+
source_mtime: sourceMtime,
|
|
1330
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1331
|
+
match_reason: matchReasonFor(mode),
|
|
1332
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1333
|
+
};
|
|
1334
|
+
})(),
|
|
1104
1335
|
}));
|
|
1105
1336
|
return {
|
|
1106
1337
|
results,
|
|
@@ -1119,31 +1350,94 @@ export class KnowledgeEngine {
|
|
|
1119
1350
|
remove(nameOrId) {
|
|
1120
1351
|
if (!this.db)
|
|
1121
1352
|
return false;
|
|
1353
|
+
if (this.disposing)
|
|
1354
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1355
|
+
if (this.activeMutations.size > 0)
|
|
1356
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1122
1357
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1123
1358
|
if (!kb)
|
|
1124
1359
|
return false;
|
|
1125
1360
|
deleteKB(this.db, kb.id);
|
|
1361
|
+
this.deleteVectorFile(kb.id);
|
|
1126
1362
|
return true;
|
|
1127
1363
|
}
|
|
1128
|
-
list() {
|
|
1364
|
+
list(signal) {
|
|
1129
1365
|
if (!this.db)
|
|
1130
1366
|
return [];
|
|
1367
|
+
throwIfAborted(signal);
|
|
1131
1368
|
return listKBs(this.db);
|
|
1132
1369
|
}
|
|
1133
1370
|
clear() {
|
|
1134
1371
|
if (!this.db)
|
|
1135
1372
|
return;
|
|
1136
|
-
|
|
1373
|
+
if (this.disposing)
|
|
1374
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1375
|
+
if (this.activeMutations.size > 0)
|
|
1376
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1377
|
+
for (const kb of listKBs(this.db)) {
|
|
1137
1378
|
deleteKB(this.db, kb.id);
|
|
1379
|
+
this.deleteVectorFile(kb.id);
|
|
1380
|
+
}
|
|
1138
1381
|
}
|
|
1139
|
-
diagnose() {
|
|
1382
|
+
diagnose(signal) {
|
|
1140
1383
|
if (!this.db)
|
|
1141
1384
|
return [];
|
|
1385
|
+
throwIfAborted(signal);
|
|
1386
|
+
const db = this.db;
|
|
1387
|
+
return listKBs(db).map((kb) => {
|
|
1388
|
+
throwIfAborted(signal);
|
|
1389
|
+
return diagnoseKB(db, kb, signal);
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
symbolSearch(query, options = {}, signal) {
|
|
1393
|
+
if (!this.db)
|
|
1394
|
+
throw new Error("Engine not initialized");
|
|
1395
|
+
throwIfAborted(signal);
|
|
1142
1396
|
const db = this.db;
|
|
1143
|
-
|
|
1397
|
+
const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
|
|
1398
|
+
if (options.kb_id && !selectedKB)
|
|
1399
|
+
throw new Error(`Knowledge base not found: ${options.kb_id}`);
|
|
1400
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1401
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
1402
|
+
const symbolOptions = {
|
|
1403
|
+
kbId: selectedKB?.id,
|
|
1404
|
+
kind: options.kind,
|
|
1405
|
+
filePattern: options.file_pattern,
|
|
1406
|
+
exact: options.exact,
|
|
1407
|
+
};
|
|
1408
|
+
const symbols = searchSymbols(db, query, {
|
|
1409
|
+
...symbolOptions,
|
|
1410
|
+
limit: limit + 1,
|
|
1411
|
+
offset,
|
|
1412
|
+
});
|
|
1413
|
+
throwIfAborted(signal);
|
|
1414
|
+
const page = symbols.slice(0, limit);
|
|
1415
|
+
const total = countSymbols(db, query, symbolOptions);
|
|
1416
|
+
throwIfAborted(signal);
|
|
1417
|
+
return {
|
|
1418
|
+
results: page.map((symbol) => {
|
|
1419
|
+
const kb = getKB(db, symbol.kb_id);
|
|
1420
|
+
return {
|
|
1421
|
+
name: symbol.name,
|
|
1422
|
+
kind: symbol.kind,
|
|
1423
|
+
file_path: symbol.file_path,
|
|
1424
|
+
file_type: symbol.file_type,
|
|
1425
|
+
kb_name: kb?.name ?? "unknown",
|
|
1426
|
+
start_line: symbol.start_line,
|
|
1427
|
+
end_line: symbol.end_line,
|
|
1428
|
+
signature: symbol.signature ?? undefined,
|
|
1429
|
+
container_name: symbol.container_name ?? undefined,
|
|
1430
|
+
text: symbol.text,
|
|
1431
|
+
indexed_at: symbol.indexed_at,
|
|
1432
|
+
};
|
|
1433
|
+
}),
|
|
1434
|
+
total_count: total,
|
|
1435
|
+
has_more: symbols.length > limit,
|
|
1436
|
+
};
|
|
1144
1437
|
}
|
|
1145
|
-
doctor() {
|
|
1146
|
-
|
|
1438
|
+
doctor(signal) {
|
|
1439
|
+
throwIfAborted(signal);
|
|
1440
|
+
const diagnostics = this.diagnose(signal);
|
|
1147
1441
|
const issues = [];
|
|
1148
1442
|
const kbs = this.list();
|
|
1149
1443
|
if (kbs.length === 0) {
|
|
@@ -1151,9 +1445,11 @@ export class KnowledgeEngine {
|
|
|
1151
1445
|
severity: "blocking",
|
|
1152
1446
|
message: "No knowledge bases are indexed.",
|
|
1153
1447
|
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1448
|
+
action_code: "rebuild_kb",
|
|
1154
1449
|
});
|
|
1155
1450
|
}
|
|
1156
1451
|
for (const diagnostic of diagnostics) {
|
|
1452
|
+
throwIfAborted(signal);
|
|
1157
1453
|
if (diagnostic.stuck_indexing) {
|
|
1158
1454
|
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1159
1455
|
issues.push({
|
|
@@ -1161,6 +1457,7 @@ export class KnowledgeEngine {
|
|
|
1161
1457
|
kb_name: diagnostic.kb_name,
|
|
1162
1458
|
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1163
1459
|
action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1460
|
+
action_code: "wait_for_indexing",
|
|
1164
1461
|
});
|
|
1165
1462
|
}
|
|
1166
1463
|
if (diagnostic.status === "error") {
|
|
@@ -1169,6 +1466,7 @@ export class KnowledgeEngine {
|
|
|
1169
1466
|
kb_name: diagnostic.kb_name,
|
|
1170
1467
|
message: "Knowledge base is in error state and is skipped by search.",
|
|
1171
1468
|
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1469
|
+
action_code: "rebuild_kb",
|
|
1172
1470
|
});
|
|
1173
1471
|
}
|
|
1174
1472
|
if (diagnostic.coverage_percent < 80) {
|
|
@@ -1177,6 +1475,7 @@ export class KnowledgeEngine {
|
|
|
1177
1475
|
kb_name: diagnostic.kb_name,
|
|
1178
1476
|
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1179
1477
|
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1478
|
+
action_code: "review_skipped_scope",
|
|
1180
1479
|
});
|
|
1181
1480
|
}
|
|
1182
1481
|
if (diagnostic.stale_files.length > 0) {
|
|
@@ -1185,6 +1484,7 @@ export class KnowledgeEngine {
|
|
|
1185
1484
|
kb_name: diagnostic.kb_name,
|
|
1186
1485
|
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1187
1486
|
action: "Run knowledge_update for this KB.",
|
|
1487
|
+
action_code: "run_update",
|
|
1188
1488
|
});
|
|
1189
1489
|
}
|
|
1190
1490
|
if (diagnostic.orphan_files.length > 0) {
|
|
@@ -1193,6 +1493,7 @@ export class KnowledgeEngine {
|
|
|
1193
1493
|
kb_name: diagnostic.kb_name,
|
|
1194
1494
|
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1195
1495
|
action: "Run knowledge_update or rebuild the KB.",
|
|
1496
|
+
action_code: "run_update",
|
|
1196
1497
|
});
|
|
1197
1498
|
}
|
|
1198
1499
|
if (diagnostic.skipped_files.total > 0) {
|
|
@@ -1204,9 +1505,24 @@ export class KnowledgeEngine {
|
|
|
1204
1505
|
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1205
1506
|
.join(", ")}).`,
|
|
1206
1507
|
action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1508
|
+
action_code: "review_skipped_scope",
|
|
1207
1509
|
});
|
|
1208
1510
|
}
|
|
1209
1511
|
}
|
|
1512
|
+
if (this.db) {
|
|
1513
|
+
for (const kb of kbs) {
|
|
1514
|
+
throwIfAborted(signal);
|
|
1515
|
+
if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
|
|
1516
|
+
issues.push({
|
|
1517
|
+
severity: "info",
|
|
1518
|
+
kb_name: kb.name,
|
|
1519
|
+
message: "No lightweight symbols are indexed for this KB.",
|
|
1520
|
+
action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
|
|
1521
|
+
action_code: "run_update",
|
|
1522
|
+
});
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1210
1526
|
const penalty = issues.reduce((score, issue) => {
|
|
1211
1527
|
if (issue.severity === "blocking")
|
|
1212
1528
|
return score + 35;
|
|
@@ -1225,135 +1541,193 @@ export class KnowledgeEngine {
|
|
|
1225
1541
|
summary,
|
|
1226
1542
|
issues,
|
|
1227
1543
|
diagnostics,
|
|
1544
|
+
actions: issues.map((issue) => ({
|
|
1545
|
+
code: issue.action_code ?? "none",
|
|
1546
|
+
kb_name: issue.kb_name,
|
|
1547
|
+
target: issue.kb_name,
|
|
1548
|
+
description: issue.action,
|
|
1549
|
+
})),
|
|
1228
1550
|
};
|
|
1229
1551
|
}
|
|
1230
|
-
async exportKB(nameOrId, outputPath) {
|
|
1552
|
+
async exportKB(nameOrId, outputPath, signal, onProgress) {
|
|
1231
1553
|
if (!this.db)
|
|
1232
1554
|
throw new Error("Engine not initialized");
|
|
1555
|
+
throwIfAborted(signal);
|
|
1233
1556
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1234
1557
|
if (!kb)
|
|
1235
1558
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
1236
|
-
|
|
1237
|
-
|
|
1559
|
+
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
|
|
1560
|
+
}
|
|
1561
|
+
async exportKBUnlocked(kb, outputPath, signal, onProgress) {
|
|
1562
|
+
if (!this.db)
|
|
1563
|
+
throw new Error("Engine not initialized");
|
|
1564
|
+
const { createWriteStream } = await import("node:fs");
|
|
1565
|
+
const tempOutputPath = tempVectorPath(outputPath);
|
|
1566
|
+
const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
|
|
1567
|
+
let count = 0;
|
|
1238
1568
|
const header = JSON.stringify({
|
|
1239
1569
|
name: kb.name,
|
|
1240
1570
|
description: kb.description,
|
|
1241
1571
|
source_type: "text",
|
|
1242
|
-
chunk_count:
|
|
1572
|
+
chunk_count: kb.chunk_count,
|
|
1243
1573
|
});
|
|
1244
|
-
|
|
1245
|
-
header
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1574
|
+
try {
|
|
1575
|
+
await writeLine(stream, header);
|
|
1576
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1577
|
+
throwIfAborted(signal);
|
|
1578
|
+
await writeLine(stream, JSON.stringify({
|
|
1579
|
+
content: chunk.content,
|
|
1580
|
+
file_path: chunk.file_path,
|
|
1581
|
+
file_type: chunk.file_type,
|
|
1582
|
+
start_line: chunk.start_line,
|
|
1583
|
+
end_line: chunk.end_line,
|
|
1584
|
+
metadata_json: chunk.metadata_json,
|
|
1585
|
+
}));
|
|
1586
|
+
count++;
|
|
1587
|
+
if (count % INDEX_EMBED_BATCH_SIZE === 0)
|
|
1588
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
|
|
1589
|
+
}
|
|
1590
|
+
await finishWriteStream(stream);
|
|
1591
|
+
throwIfAborted(signal);
|
|
1592
|
+
renameSync(tempOutputPath, outputPath);
|
|
1593
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
|
|
1594
|
+
return count;
|
|
1595
|
+
}
|
|
1596
|
+
catch (error) {
|
|
1597
|
+
stream.destroy();
|
|
1598
|
+
rmSync(tempOutputPath, { force: true });
|
|
1599
|
+
throw error;
|
|
1600
|
+
}
|
|
1257
1601
|
}
|
|
1258
1602
|
async importKB(inputPath, onProgress, signal) {
|
|
1259
1603
|
if (!this.db)
|
|
1260
1604
|
throw new Error("Engine not initialized");
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1605
|
+
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
|
|
1606
|
+
}
|
|
1607
|
+
async importKBUnlocked(inputPath, onProgress, signal) {
|
|
1608
|
+
if (!this.db)
|
|
1609
|
+
throw new Error("Engine not initialized");
|
|
1610
|
+
throwIfAborted(signal);
|
|
1611
|
+
const { createReadStream } = await import("node:fs");
|
|
1612
|
+
const { createInterface } = await import("node:readline");
|
|
1613
|
+
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1614
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1615
|
+
let header;
|
|
1616
|
+
let kb;
|
|
1269
1617
|
let vectorWriter;
|
|
1270
1618
|
let tempVectorFile;
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1619
|
+
let inserted = 0;
|
|
1620
|
+
let lineNumber = 0;
|
|
1621
|
+
const pendingChunks = [];
|
|
1622
|
+
const pendingSymbols = [];
|
|
1623
|
+
const flushPending = async () => {
|
|
1624
|
+
if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
|
|
1625
|
+
return;
|
|
1626
|
+
throwIfAborted(signal);
|
|
1627
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
1628
|
+
const symbols = pendingSymbols.splice(0);
|
|
1629
|
+
const total = header?.chunk_count;
|
|
1630
|
+
const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
|
|
1274
1631
|
updateIndexingJob(this.db, kb.id, {
|
|
1275
|
-
phase: "
|
|
1276
|
-
message
|
|
1277
|
-
|
|
1632
|
+
phase: "embedding",
|
|
1633
|
+
message,
|
|
1634
|
+
processed_chunks: inserted,
|
|
1635
|
+
total_files: total,
|
|
1636
|
+
added_chunks: inserted,
|
|
1278
1637
|
});
|
|
1279
|
-
onProgress?.(
|
|
1280
|
-
const
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1638
|
+
onProgress?.(message);
|
|
1639
|
+
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1640
|
+
throwIfAborted(signal);
|
|
1641
|
+
insertChunks(this.db, kb.id, batch);
|
|
1642
|
+
insertSymbols(this.db, kb.id, symbols);
|
|
1643
|
+
vectorWriter.append(vectors);
|
|
1644
|
+
inserted += batch.length;
|
|
1645
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1646
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1647
|
+
phase: "storing",
|
|
1648
|
+
message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
|
|
1649
|
+
processed_chunks: inserted,
|
|
1650
|
+
total_files: total,
|
|
1651
|
+
added_chunks: inserted,
|
|
1652
|
+
});
|
|
1653
|
+
};
|
|
1654
|
+
try {
|
|
1655
|
+
for await (const rawLine of lines) {
|
|
1656
|
+
throwIfAborted(signal);
|
|
1657
|
+
lineNumber++;
|
|
1658
|
+
const line = rawLine.trim();
|
|
1659
|
+
if (!line)
|
|
1660
|
+
continue;
|
|
1661
|
+
if (!header) {
|
|
1662
|
+
header = JSON.parse(line);
|
|
1663
|
+
if (!header.name)
|
|
1664
|
+
throw new Error("Import header is missing a knowledge base name");
|
|
1665
|
+
const existingKB = getKBByName(this.db, header.name);
|
|
1666
|
+
if (existingKB) {
|
|
1667
|
+
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
1668
|
+
}
|
|
1669
|
+
kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
1670
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
1671
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1672
|
+
const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
|
|
1673
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1674
|
+
phase: "importing",
|
|
1675
|
+
message: importMessage,
|
|
1676
|
+
total_files: header.chunk_count,
|
|
1677
|
+
});
|
|
1678
|
+
onProgress?.(importMessage);
|
|
1679
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
1680
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1681
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1682
|
+
continue;
|
|
1683
|
+
}
|
|
1684
|
+
const imported = JSON.parse(line);
|
|
1685
|
+
const chunk = importedChunkToInsert(imported);
|
|
1686
|
+
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
1687
|
+
pendingChunks.push(chunk);
|
|
1688
|
+
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
1689
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
|
|
1690
|
+
await flushPending();
|
|
1332
1691
|
}
|
|
1692
|
+
if (!header)
|
|
1693
|
+
throw new Error("Empty import file");
|
|
1694
|
+
if (!kb || !vectorWriter || !tempVectorFile)
|
|
1695
|
+
throw new Error("Import did not create a knowledge base");
|
|
1696
|
+
await flushPending();
|
|
1333
1697
|
vectorWriter.close();
|
|
1334
1698
|
vectorWriter = undefined;
|
|
1335
|
-
renameSync(tempVectorFile,
|
|
1699
|
+
renameSync(tempVectorFile, this.vectorPathFor(kb.id));
|
|
1336
1700
|
tempVectorFile = undefined;
|
|
1337
|
-
updateKBCounts(this.db, kb.id,
|
|
1701
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1338
1702
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1339
1703
|
const savedKB = getKB(this.db, kb.id);
|
|
1340
1704
|
if (!savedKB)
|
|
1341
1705
|
throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1342
|
-
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${
|
|
1343
|
-
return { kb: savedKB, chunkCount:
|
|
1706
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
|
|
1707
|
+
return { kb: savedKB, chunkCount: inserted };
|
|
1344
1708
|
}
|
|
1345
|
-
catch (
|
|
1709
|
+
catch (error) {
|
|
1710
|
+
lines.close();
|
|
1711
|
+
stream.destroy();
|
|
1346
1712
|
vectorWriter?.close();
|
|
1347
1713
|
if (tempVectorFile)
|
|
1348
1714
|
rmSync(tempVectorFile, { force: true });
|
|
1349
|
-
if (this.db) {
|
|
1350
|
-
finishIndexingJob(this.db, kb.id, isCancellationError(
|
|
1715
|
+
if (this.db && kb) {
|
|
1716
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
|
|
1717
|
+
deleteKB(this.db, kb.id);
|
|
1718
|
+
this.deleteVectorFile(kb.id);
|
|
1351
1719
|
}
|
|
1352
|
-
|
|
1353
|
-
throw e;
|
|
1720
|
+
throw error;
|
|
1354
1721
|
}
|
|
1355
1722
|
}
|
|
1356
1723
|
async dispose(options = {}) {
|
|
1724
|
+
this.disposing = true;
|
|
1725
|
+
const activeUpdates = [...this.activeUpdates.values()];
|
|
1726
|
+
if (activeUpdates.length > 0)
|
|
1727
|
+
await Promise.allSettled(activeUpdates);
|
|
1728
|
+
const activeMutations = [...this.activeMutations.values()];
|
|
1729
|
+
if (activeMutations.length > 0)
|
|
1730
|
+
await Promise.allSettled(activeMutations);
|
|
1357
1731
|
const disposeModels = options.disposeModels ?? true;
|
|
1358
1732
|
if (disposeModels) {
|
|
1359
1733
|
await disposeEmbedding();
|