pi-knowledge 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +9 -2
- package/dist/index.js +129 -20
- package/dist/src/diagnostics/health.d.ts +1 -1
- package/dist/src/diagnostics/health.js +12 -1
- package/dist/src/embedding/provider.d.ts +1 -1
- package/dist/src/embedding/provider.js +12 -4
- package/dist/src/engine.d.ts +61 -10
- package/dist/src/engine.js +519 -165
- package/dist/src/indexer/symbols.d.ts +2 -0
- package/dist/src/indexer/symbols.js +175 -0
- package/dist/src/model-worker-client.d.ts +1 -1
- package/dist/src/model-worker-client.js +7 -7
- package/dist/src/search/reranker.d.ts +1 -1
- package/dist/src/search/reranker.js +2 -2
- package/dist/src/search/vector.d.ts +1 -1
- package/dist/src/search/vector.js +5 -1
- package/dist/src/storage/sqlite.d.ts +36 -1
- package/dist/src/storage/sqlite.js +159 -2
- package/dist/src/watcher/file-watcher.d.ts +2 -1
- package/dist/src/watcher/file-watcher.js +20 -30
- package/docs/configuration.md +8 -2
- package/package.json +1 -1
package/dist/src/engine.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { existsSync, renameSync, rmSync, statSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync, renameSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { join, resolve } from "node:path";
|
|
3
3
|
import { diagnoseKB } from "./diagnostics/health.js";
|
|
4
4
|
import { dispose as disposeEmbedding, embedDocuments, embedQuery, prepareForShutdown as prepareEmbeddingForShutdown, } from "./embedding/provider.js";
|
|
5
5
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
6
|
import { buildChunkEmbeddingText, chunkFile, chunkIdentityHash, createSkippedScanStats, isReadableTextFile, iterateScannableFiles, iterateScannedFiles, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
|
+
import { extractSymbols } from "./indexer/symbols.js";
|
|
7
8
|
import { shutdownModelWorker } from "./model-worker-client.js";
|
|
8
9
|
import { searchBM25 } from "./search/bm25.js";
|
|
9
10
|
import { weightedScoreFusion } from "./search/fusion.js";
|
|
@@ -11,7 +12,7 @@ import { normalizedQueryText, tokenizeForSearch } from "./search/query.js";
|
|
|
11
12
|
import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, MIN_HYBRID_SCORE, normalizeFileTypeFilter, queryCoverage, scoreChunkForQuery, } from "./search/ranking.js";
|
|
12
13
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
13
14
|
import { searchVectorFile } from "./search/vector.js";
|
|
14
|
-
import { createKB, deleteChunksByIds, deleteKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile,
|
|
15
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBStatus, } from "./storage/sqlite.js";
|
|
15
16
|
export const CURRENT_EMBEDDING_MODEL = "multilingual-e5-small";
|
|
16
17
|
const ADAPTIVE_CONTEXT_LINES = 80;
|
|
17
18
|
const ADAPTIVE_MAX_CONTEXT_CHARS = 6_000;
|
|
@@ -24,6 +25,26 @@ function isCancellationError(error) {
|
|
|
24
25
|
function tempVectorPath(vectorPath) {
|
|
25
26
|
return `${vectorPath}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
26
27
|
}
|
|
28
|
+
function retrievalModeFor(mode) {
|
|
29
|
+
if (mode === "auto")
|
|
30
|
+
return "hybrid";
|
|
31
|
+
if (mode === "code" || mode === "config" || mode === "errors")
|
|
32
|
+
return "fast";
|
|
33
|
+
if (mode === "docs" || mode === "decision")
|
|
34
|
+
return "hybrid";
|
|
35
|
+
return mode;
|
|
36
|
+
}
|
|
37
|
+
function matchReasonFor(mode) {
|
|
38
|
+
if (mode === "deep")
|
|
39
|
+
return "rerank";
|
|
40
|
+
if (mode === "adaptive")
|
|
41
|
+
return "adaptive";
|
|
42
|
+
if (mode === "semantic")
|
|
43
|
+
return "vector";
|
|
44
|
+
if (mode === "fast" || mode === "code" || mode === "config" || mode === "errors")
|
|
45
|
+
return "bm25";
|
|
46
|
+
return "hybrid";
|
|
47
|
+
}
|
|
27
48
|
function toScanOptions(options = {}) {
|
|
28
49
|
return {
|
|
29
50
|
includeSuggestedText: options.include_suggested_text === true,
|
|
@@ -60,11 +81,12 @@ function parseAddOptions(raw) {
|
|
|
60
81
|
return {};
|
|
61
82
|
}
|
|
62
83
|
}
|
|
63
|
-
function planDirectoryScan(dirPath, options = {}) {
|
|
84
|
+
function planDirectoryScan(dirPath, options = {}, signal) {
|
|
64
85
|
const skipped = createSkippedScanStats();
|
|
65
86
|
let files = 0;
|
|
66
87
|
let bytes = 0;
|
|
67
88
|
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
89
|
+
throwIfAborted(signal);
|
|
68
90
|
files++;
|
|
69
91
|
bytes += file.size;
|
|
70
92
|
}
|
|
@@ -76,6 +98,25 @@ function planDirectoryScan(dirPath, options = {}) {
|
|
|
76
98
|
skipped,
|
|
77
99
|
};
|
|
78
100
|
}
|
|
101
|
+
function sourceMtimeFor(kb, filePath) {
|
|
102
|
+
if (!kb?.source_path)
|
|
103
|
+
return undefined;
|
|
104
|
+
const absPath = kb.source_type === "directory" ? join(kb.source_path, filePath) : kb.source_path;
|
|
105
|
+
try {
|
|
106
|
+
return statSync(absPath).mtimeMs;
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function kbTrustMultiplier(kb) {
|
|
113
|
+
let multiplier = kb.status === "stale" ? 0.92 : 1;
|
|
114
|
+
if (kb.source_type === "directory" || kb.source_type === "file")
|
|
115
|
+
multiplier *= 1.04;
|
|
116
|
+
if (kb.source_type === "text" || kb.source_type === "url")
|
|
117
|
+
multiplier *= 0.98;
|
|
118
|
+
return multiplier;
|
|
119
|
+
}
|
|
79
120
|
function formatBytes(bytes) {
|
|
80
121
|
if (bytes < 1024)
|
|
81
122
|
return `${bytes} B`;
|
|
@@ -347,22 +388,143 @@ async function chunkUrl(source, signal) {
|
|
|
347
388
|
function normalizeExtractedText(text) {
|
|
348
389
|
return Array.isArray(text) ? text.join("\n\n") : text;
|
|
349
390
|
}
|
|
391
|
+
async function extractSourceFileContent(filePath, signal) {
|
|
392
|
+
const lowerPath = filePath.toLowerCase();
|
|
393
|
+
if (lowerPath.endsWith(".pdf")) {
|
|
394
|
+
throwIfAborted(signal);
|
|
395
|
+
// PDF extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
396
|
+
const { extractText } = await import("unpdf");
|
|
397
|
+
throwIfAborted(signal);
|
|
398
|
+
const buf = readFileSync(filePath);
|
|
399
|
+
throwIfAborted(signal);
|
|
400
|
+
const { text } = await extractText(new Uint8Array(buf));
|
|
401
|
+
throwIfAborted(signal);
|
|
402
|
+
return { content: normalizeExtractedText(text), fileType: "pdf" };
|
|
403
|
+
}
|
|
404
|
+
if (lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc")) {
|
|
405
|
+
throwIfAborted(signal);
|
|
406
|
+
// DOCX extraction is an optional heavy runtime path; keep parser loading out of extension startup.
|
|
407
|
+
const mammoth = await import("mammoth");
|
|
408
|
+
throwIfAborted(signal);
|
|
409
|
+
const result = await mammoth.extractRawText({ path: filePath });
|
|
410
|
+
throwIfAborted(signal);
|
|
411
|
+
return { content: result.value, fileType: "docx" };
|
|
412
|
+
}
|
|
413
|
+
if (!isReadableTextFile(filePath)) {
|
|
414
|
+
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
415
|
+
}
|
|
416
|
+
throwIfAborted(signal);
|
|
417
|
+
return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
|
|
418
|
+
}
|
|
419
|
+
function throwIfAborted(signal) {
|
|
420
|
+
if (signal?.aborted)
|
|
421
|
+
throw new Error("Cancelled");
|
|
422
|
+
}
|
|
423
|
+
function importedChunkToInsert(chunk) {
|
|
424
|
+
const metadataJson = chunk.metadata_json || "{}";
|
|
425
|
+
return {
|
|
426
|
+
content_hash: chunkIdentityHash({
|
|
427
|
+
content: chunk.content,
|
|
428
|
+
filePath: chunk.file_path,
|
|
429
|
+
fileType: chunk.file_type,
|
|
430
|
+
startLine: chunk.start_line,
|
|
431
|
+
endLine: chunk.end_line,
|
|
432
|
+
metadataJson,
|
|
433
|
+
}),
|
|
434
|
+
content: chunk.content,
|
|
435
|
+
content_tokenized: "",
|
|
436
|
+
file_path: chunk.file_path,
|
|
437
|
+
file_type: chunk.file_type,
|
|
438
|
+
start_line: chunk.start_line,
|
|
439
|
+
end_line: chunk.end_line,
|
|
440
|
+
metadata_json: metadataJson,
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
async function writeLine(stream, line) {
|
|
444
|
+
if (stream.write(`${line}\n`))
|
|
445
|
+
return;
|
|
446
|
+
await new Promise((resolve, reject) => {
|
|
447
|
+
const cleanup = () => {
|
|
448
|
+
stream.off("drain", onDrain);
|
|
449
|
+
stream.off("error", onError);
|
|
450
|
+
};
|
|
451
|
+
const onDrain = () => {
|
|
452
|
+
cleanup();
|
|
453
|
+
resolve();
|
|
454
|
+
};
|
|
455
|
+
const onError = (error) => {
|
|
456
|
+
cleanup();
|
|
457
|
+
reject(error);
|
|
458
|
+
};
|
|
459
|
+
stream.once("drain", onDrain);
|
|
460
|
+
stream.once("error", onError);
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
async function finishWriteStream(stream) {
|
|
464
|
+
await new Promise((resolve, reject) => {
|
|
465
|
+
const cleanup = () => {
|
|
466
|
+
stream.off("finish", onFinish);
|
|
467
|
+
stream.off("error", onError);
|
|
468
|
+
};
|
|
469
|
+
const onFinish = () => {
|
|
470
|
+
cleanup();
|
|
471
|
+
resolve();
|
|
472
|
+
};
|
|
473
|
+
const onError = (error) => {
|
|
474
|
+
cleanup();
|
|
475
|
+
reject(error);
|
|
476
|
+
};
|
|
477
|
+
stream.once("finish", onFinish);
|
|
478
|
+
stream.once("error", onError);
|
|
479
|
+
stream.end();
|
|
480
|
+
});
|
|
481
|
+
}
|
|
350
482
|
export class KnowledgeEngine {
|
|
351
483
|
db = null;
|
|
352
484
|
knowledgeDir = "";
|
|
353
485
|
activeUpdates = new Map();
|
|
486
|
+
activeMutations = new Map();
|
|
487
|
+
disposing = false;
|
|
488
|
+
runExclusive(keys, description, operation) {
|
|
489
|
+
if (this.disposing)
|
|
490
|
+
throw new Error("Knowledge engine is shutting down");
|
|
491
|
+
const mutationKeys = Array.isArray(keys) ? keys : [keys];
|
|
492
|
+
for (const key of mutationKeys) {
|
|
493
|
+
if (this.activeMutations.has(key))
|
|
494
|
+
throw new Error(`${description} is already running`);
|
|
495
|
+
}
|
|
496
|
+
const run = Promise.resolve()
|
|
497
|
+
.then(operation)
|
|
498
|
+
.finally(() => {
|
|
499
|
+
for (const key of mutationKeys) {
|
|
500
|
+
if (this.activeMutations.get(key) === run)
|
|
501
|
+
this.activeMutations.delete(key);
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
for (const key of mutationKeys)
|
|
505
|
+
this.activeMutations.set(key, run);
|
|
506
|
+
return run;
|
|
507
|
+
}
|
|
354
508
|
async initialize(knowledgeDir) {
|
|
355
509
|
this.knowledgeDir = knowledgeDir;
|
|
356
510
|
this.db = openDatabase(knowledgeDir);
|
|
511
|
+
this.disposing = false;
|
|
357
512
|
}
|
|
358
|
-
|
|
513
|
+
vectorPathFor(kbId) {
|
|
514
|
+
return join(this.knowledgeDir, "vectors", `${kbId}.bin`);
|
|
515
|
+
}
|
|
516
|
+
deleteVectorFile(kbId) {
|
|
517
|
+
rmSync(this.vectorPathFor(kbId), { force: true });
|
|
518
|
+
}
|
|
519
|
+
plan(source, options = {}, signal) {
|
|
520
|
+
throwIfAborted(signal);
|
|
359
521
|
const resolvedSource = resolve(source);
|
|
360
522
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
361
523
|
const isDir = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
|
|
362
524
|
const isFile = !isUrl && existsSync(resolvedSource) && statSync(resolvedSource).isFile();
|
|
363
525
|
const sourceType = isUrl ? "url" : isDir ? "directory" : isFile ? "file" : "text";
|
|
364
526
|
if (isDir) {
|
|
365
|
-
const plan = planDirectoryScan(resolvedSource, toScanOptions(options));
|
|
527
|
+
const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
|
|
366
528
|
return {
|
|
367
529
|
source_type: "directory",
|
|
368
530
|
scannable_files: plan.files,
|
|
@@ -405,8 +567,13 @@ export class KnowledgeEngine {
|
|
|
405
567
|
};
|
|
406
568
|
}
|
|
407
569
|
async add(source, name, onProgress, signal, options = {}) {
|
|
570
|
+
throwIfAborted(signal);
|
|
571
|
+
return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options));
|
|
572
|
+
}
|
|
573
|
+
async addUnlocked(source, name, onProgress, signal, options = {}) {
|
|
408
574
|
if (!this.db)
|
|
409
575
|
throw new Error("Engine not initialized");
|
|
576
|
+
throwIfAborted(signal);
|
|
410
577
|
const db = this.db;
|
|
411
578
|
const resolvedSource = resolve(source);
|
|
412
579
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
@@ -429,7 +596,7 @@ export class KnowledgeEngine {
|
|
|
429
596
|
let vectorWriter;
|
|
430
597
|
let tempVectorFile;
|
|
431
598
|
try {
|
|
432
|
-
const vectorPath =
|
|
599
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
433
600
|
tempVectorFile = tempVectorPath(vectorPath);
|
|
434
601
|
const writer = openVectorWriter(tempVectorFile);
|
|
435
602
|
vectorWriter = writer;
|
|
@@ -488,33 +655,26 @@ export class KnowledgeEngine {
|
|
|
488
655
|
await flushPending(processedFiles, totalFiles);
|
|
489
656
|
}
|
|
490
657
|
};
|
|
658
|
+
const addSymbols = (content, filePath, fileType) => {
|
|
659
|
+
insertSymbols(db, kb.id, extractSymbols(content, filePath, fileType));
|
|
660
|
+
};
|
|
491
661
|
if (isUrl) {
|
|
492
662
|
const message = `Fetching ${source}...`;
|
|
493
663
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
494
664
|
onProgress?.(message);
|
|
495
665
|
const chunks = await chunkUrl(source, signal);
|
|
496
666
|
fileCount = 1;
|
|
667
|
+
addSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), source, "html");
|
|
497
668
|
await addChunks(chunks);
|
|
498
669
|
}
|
|
499
|
-
else if (isFile
|
|
500
|
-
|
|
501
|
-
onProgress?.("Extracting text from PDF...");
|
|
502
|
-
const { extractText } = await import("unpdf");
|
|
503
|
-
const buf = (await import("node:fs")).readFileSync(resolvedSource);
|
|
504
|
-
const { text } = await extractText(new Uint8Array(buf));
|
|
505
|
-
fileCount = 1;
|
|
506
|
-
await addChunks(await chunkFile(normalizeExtractedText(text), resolvedSource));
|
|
507
|
-
}
|
|
508
|
-
else if (isFile && (resolvedSource.endsWith(".docx") || resolvedSource.endsWith(".doc"))) {
|
|
509
|
-
updateIndexingJob(this.db, kb.id, { phase: "extracting", message: "Extracting text from DOCX..." });
|
|
510
|
-
onProgress?.("Extracting text from DOCX...");
|
|
511
|
-
const mammoth = await import("mammoth");
|
|
512
|
-
const result = await mammoth.extractRawText({ path: resolvedSource });
|
|
670
|
+
else if (isFile) {
|
|
671
|
+
const extracted = await extractSourceFileContent(resolvedSource, signal);
|
|
513
672
|
fileCount = 1;
|
|
514
|
-
|
|
673
|
+
addSymbols(extracted.content, resolvedSource, extracted.fileType);
|
|
674
|
+
await addChunks(await chunkFile(extracted.content, resolvedSource));
|
|
515
675
|
}
|
|
516
676
|
else if (isDir) {
|
|
517
|
-
const plan = planDirectoryScan(resolvedSource, scanOptions);
|
|
677
|
+
const plan = planDirectoryScan(resolvedSource, scanOptions, signal);
|
|
518
678
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
519
679
|
updateIndexingJob(this.db, kb.id, {
|
|
520
680
|
phase: "planning",
|
|
@@ -541,6 +701,7 @@ export class KnowledgeEngine {
|
|
|
541
701
|
latestSkippedTotal = skipped.total;
|
|
542
702
|
if (chunks.length > 0)
|
|
543
703
|
fileCount++;
|
|
704
|
+
addSymbols(file.content, file.relPath, file.fileType);
|
|
544
705
|
await addChunks(chunks, processedFiles, plan.files);
|
|
545
706
|
if (processedFiles % 25 === 0)
|
|
546
707
|
reportProgress("Chunking", processedFiles, plan.files, skipped.total);
|
|
@@ -557,17 +718,9 @@ export class KnowledgeEngine {
|
|
|
557
718
|
});
|
|
558
719
|
onProgress?.(finalizingMessage);
|
|
559
720
|
}
|
|
560
|
-
else if (isFile) {
|
|
561
|
-
if (!isReadableTextFile(resolvedSource)) {
|
|
562
|
-
throw new Error(`File is not readable text and has no supported extractor: ${resolvedSource}`);
|
|
563
|
-
}
|
|
564
|
-
const { readFileSync } = await import("node:fs");
|
|
565
|
-
const content = readFileSync(resolvedSource, "utf-8");
|
|
566
|
-
fileCount = 1;
|
|
567
|
-
await addChunks(await chunkFile(content, resolvedSource));
|
|
568
|
-
}
|
|
569
721
|
else {
|
|
570
722
|
fileCount = 1;
|
|
723
|
+
addSymbols(source, "inline-text", "text");
|
|
571
724
|
await addChunks(await chunkFile(source, "inline-text"));
|
|
572
725
|
}
|
|
573
726
|
await flushPending();
|
|
@@ -600,15 +753,18 @@ export class KnowledgeEngine {
|
|
|
600
753
|
rmSync(tempVectorFile, { force: true });
|
|
601
754
|
if (this.db) {
|
|
602
755
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Indexing cancelled." : "Indexing failed.", e instanceof Error ? e.message : String(e));
|
|
756
|
+
deleteKB(this.db, kb.id);
|
|
757
|
+
this.deleteVectorFile(kb.id);
|
|
603
758
|
}
|
|
604
|
-
// Clean up partial state on failure
|
|
605
|
-
deleteKB(this.db, kb.id);
|
|
606
759
|
throw e;
|
|
607
760
|
}
|
|
608
761
|
}
|
|
609
762
|
async update(nameOrId, onProgress, signal) {
|
|
610
763
|
if (!this.db)
|
|
611
764
|
throw new Error("Engine not initialized");
|
|
765
|
+
if (this.disposing)
|
|
766
|
+
throw new Error("Knowledge engine is shutting down");
|
|
767
|
+
throwIfAborted(signal);
|
|
612
768
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
613
769
|
if (!kb)
|
|
614
770
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
@@ -621,7 +777,7 @@ export class KnowledgeEngine {
|
|
|
621
777
|
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
622
778
|
return activeUpdate;
|
|
623
779
|
}
|
|
624
|
-
const updateRun = this.runUpdate(updateableKB, onProgress, signal).finally(() => {
|
|
780
|
+
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
|
|
625
781
|
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
626
782
|
this.activeUpdates.delete(updateableKB.id);
|
|
627
783
|
});
|
|
@@ -631,14 +787,17 @@ export class KnowledgeEngine {
|
|
|
631
787
|
async runUpdate(kb, onProgress, signal) {
|
|
632
788
|
if (!this.db)
|
|
633
789
|
throw new Error("Engine not initialized");
|
|
790
|
+
throwIfAborted(signal);
|
|
634
791
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
635
792
|
startIndexingJob(this.db, kb.id, "update", `Starting update for "${kb.name}"`);
|
|
636
793
|
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
637
794
|
let replacementVectorPath;
|
|
638
795
|
let addedVectorPath;
|
|
639
796
|
let addedVectorWriter;
|
|
797
|
+
const insertedChunkIds = [];
|
|
798
|
+
const stagedSymbols = [];
|
|
640
799
|
try {
|
|
641
|
-
const vectorPath =
|
|
800
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
642
801
|
addedVectorPath = tempVectorPath(`${vectorPath}.added`);
|
|
643
802
|
addedVectorWriter = openVectorWriter(addedVectorPath);
|
|
644
803
|
const existingHashes = new Map();
|
|
@@ -687,7 +846,7 @@ export class KnowledgeEngine {
|
|
|
687
846
|
newVectorIndexByHash.set(batch[i].content_hash, indexes);
|
|
688
847
|
}
|
|
689
848
|
addedVectorCount += newVectors.length;
|
|
690
|
-
insertChunks(this.db, kb.id, batch);
|
|
849
|
+
insertedChunkIds.push(...insertChunks(this.db, kb.id, batch));
|
|
691
850
|
addedCount += batch.length;
|
|
692
851
|
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
693
852
|
const storedMessage = `Stored update batch: +${addedCount} chunks, =${unchanged} unchanged`;
|
|
@@ -728,10 +887,12 @@ export class KnowledgeEngine {
|
|
|
728
887
|
updateIndexingJob(this.db, kb.id, { phase: "fetching", message });
|
|
729
888
|
onProgress?.(message);
|
|
730
889
|
scannedFiles = 1;
|
|
731
|
-
|
|
890
|
+
const chunks = await chunkUrl(kb.source_path, signal);
|
|
891
|
+
stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
|
|
892
|
+
await processChunks(chunks);
|
|
732
893
|
}
|
|
733
894
|
else if (statSync(kb.source_path).isDirectory()) {
|
|
734
|
-
const plan = planDirectoryScan(kb.source_path, scanOptions);
|
|
895
|
+
const plan = planDirectoryScan(kb.source_path, scanOptions, signal);
|
|
735
896
|
plannedTotalFiles = plan.files;
|
|
736
897
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
737
898
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -746,6 +907,7 @@ export class KnowledgeEngine {
|
|
|
746
907
|
if (signal?.aborted)
|
|
747
908
|
throw new Error("Cancelled");
|
|
748
909
|
scannedFiles++;
|
|
910
|
+
stagedSymbols.push(...extractSymbols(file.content, file.relPath, file.fileType));
|
|
749
911
|
await processChunks(await chunkFile(file.content, file.relPath));
|
|
750
912
|
if (scannedFiles % 25 === 0) {
|
|
751
913
|
const message = `Scanned ${scannedFiles} files, ${scannedChunks} chunks, skipped ${skipped.total}, +${addedCount} =${unchanged}`;
|
|
@@ -776,9 +938,10 @@ export class KnowledgeEngine {
|
|
|
776
938
|
onProgress?.(message);
|
|
777
939
|
}
|
|
778
940
|
else {
|
|
779
|
-
const
|
|
941
|
+
const extracted = await extractSourceFileContent(kb.source_path, signal);
|
|
780
942
|
scannedFiles = 1;
|
|
781
|
-
|
|
943
|
+
stagedSymbols.push(...extractSymbols(extracted.content, kb.source_path, extracted.fileType));
|
|
944
|
+
await processChunks(await chunkFile(extracted.content, kb.source_path));
|
|
782
945
|
}
|
|
783
946
|
await flushPending();
|
|
784
947
|
addedVectorWriter.close();
|
|
@@ -787,8 +950,7 @@ export class KnowledgeEngine {
|
|
|
787
950
|
for (const entries of existingHashes.values()) {
|
|
788
951
|
idsToRemove.push(...entries.map((entry) => entry.id));
|
|
789
952
|
}
|
|
790
|
-
|
|
791
|
-
deleteChunksByIds(this.db, idsToRemove);
|
|
953
|
+
const idsToRemoveSet = new Set(idsToRemove);
|
|
792
954
|
const changesMessage = `Changes: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
793
955
|
updateIndexingJob(this.db, kb.id, {
|
|
794
956
|
phase: "reconciling",
|
|
@@ -811,6 +973,10 @@ export class KnowledgeEngine {
|
|
|
811
973
|
};
|
|
812
974
|
try {
|
|
813
975
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
976
|
+
if (signal?.aborted)
|
|
977
|
+
throw new Error("Cancelled");
|
|
978
|
+
if (idsToRemoveSet.has(chunk.id))
|
|
979
|
+
continue;
|
|
814
980
|
const oldVectorIndex = takeVectorIndex(oldVectorIndexByHash, chunk.content_hash);
|
|
815
981
|
const newVectorIndex = takeVectorIndex(newVectorIndexByHash, chunk.content_hash);
|
|
816
982
|
const vector = oldVectorReader && oldVectorIndex !== undefined
|
|
@@ -847,6 +1013,10 @@ export class KnowledgeEngine {
|
|
|
847
1013
|
if (addedVectorPath)
|
|
848
1014
|
rmSync(addedVectorPath, { force: true });
|
|
849
1015
|
addedVectorPath = undefined;
|
|
1016
|
+
if (idsToRemove.length > 0)
|
|
1017
|
+
deleteChunksByIds(this.db, idsToRemove);
|
|
1018
|
+
deleteSymbolsByKB(this.db, kb.id);
|
|
1019
|
+
insertSymbols(this.db, kb.id, stagedSymbols);
|
|
850
1020
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
851
1021
|
updateKBStatus(this.db, kb.id, "ready");
|
|
852
1022
|
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}`;
|
|
@@ -869,14 +1039,19 @@ export class KnowledgeEngine {
|
|
|
869
1039
|
rmSync(replacementVectorPath, { force: true });
|
|
870
1040
|
if (addedVectorPath)
|
|
871
1041
|
rmSync(addedVectorPath, { force: true });
|
|
872
|
-
|
|
1042
|
+
if (insertedChunkIds.length > 0) {
|
|
1043
|
+
deleteChunksByIds(this.db, insertedChunkIds);
|
|
1044
|
+
updateKBCounts(this.db, kb.id, getChunkCount(this.db, kb.id), getFileCount(this.db, kb.id));
|
|
1045
|
+
}
|
|
1046
|
+
updateKBStatus(this.db, kb.id, isCancellationError(e) ? kb.status : "error");
|
|
873
1047
|
finishIndexingJob(this.db, kb.id, isCancellationError(e) ? "cancelled" : "failed", isCancellationError(e) ? "Update cancelled." : "Update failed.", e instanceof Error ? e.message : String(e));
|
|
874
1048
|
throw e;
|
|
875
1049
|
}
|
|
876
1050
|
}
|
|
877
|
-
async search(query, options = {}) {
|
|
1051
|
+
async search(query, options = {}, signal) {
|
|
878
1052
|
if (!this.db)
|
|
879
1053
|
throw new Error("Engine not initialized");
|
|
1054
|
+
throwIfAborted(signal);
|
|
880
1055
|
const requestedMode = options.mode ?? "hybrid";
|
|
881
1056
|
if (requestedMode === "auto") {
|
|
882
1057
|
const primaryMode = chooseAutoMode(query);
|
|
@@ -887,7 +1062,8 @@ export class KnowledgeEngine {
|
|
|
887
1062
|
if (tried.includes(mode))
|
|
888
1063
|
continue;
|
|
889
1064
|
tried.push(mode);
|
|
890
|
-
|
|
1065
|
+
throwIfAborted(signal);
|
|
1066
|
+
const response = await this.search(query, { ...options, mode }, signal);
|
|
891
1067
|
if (response.warnings)
|
|
892
1068
|
warnings.push(...response.warnings);
|
|
893
1069
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
@@ -916,7 +1092,8 @@ export class KnowledgeEngine {
|
|
|
916
1092
|
}
|
|
917
1093
|
const db = this.db;
|
|
918
1094
|
const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters, diversity = "balanced" } = options;
|
|
919
|
-
const
|
|
1095
|
+
const resolvedMode = retrievalModeFor(mode);
|
|
1096
|
+
const retrievalMode = resolvedMode === "adaptive" ? "hybrid" : resolvedMode;
|
|
920
1097
|
const candidateLimit = Math.max(50, offset + limit * 12);
|
|
921
1098
|
const normalizedQuery = normalizedQueryText(query);
|
|
922
1099
|
const queryTokens = tokenizeForSimilarity(query);
|
|
@@ -925,6 +1102,7 @@ export class KnowledgeEngine {
|
|
|
925
1102
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
926
1103
|
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : listKBs(db);
|
|
927
1104
|
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1105
|
+
const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
|
|
928
1106
|
for (const kb of availableKBs) {
|
|
929
1107
|
if (kb.status !== "ready" && kb.status !== "stale") {
|
|
930
1108
|
warnings.push(`"${kb.name}" is ${kb.status}; search skipped until indexing is ready`);
|
|
@@ -942,19 +1120,24 @@ export class KnowledgeEngine {
|
|
|
942
1120
|
const allResults = [];
|
|
943
1121
|
const vectorsByChunkId = new Map();
|
|
944
1122
|
for (const kb of kbs) {
|
|
1123
|
+
throwIfAborted(signal);
|
|
945
1124
|
if (kb.embedding_model !== CURRENT_EMBEDDING_MODEL) {
|
|
946
1125
|
warnings.push(`"${kb.name}" was indexed with ${kb.embedding_model} (current: ${CURRENT_EMBEDDING_MODEL}) — run knowledge_update for best results`);
|
|
947
1126
|
}
|
|
948
1127
|
if (kb.chunk_count === 0)
|
|
949
1128
|
continue;
|
|
950
|
-
const vectorPath =
|
|
1129
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
951
1130
|
if (retrievalMode === "fast") {
|
|
952
|
-
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }))
|
|
1131
|
+
allResults.push(...searchBM25(db, normalizedQuery || query, candidateLimit, kb.id, { allowOrFallback: false }).map((result) => ({
|
|
1132
|
+
...result,
|
|
1133
|
+
score: result.score * kbTrustMultiplier(kb),
|
|
1134
|
+
})));
|
|
953
1135
|
}
|
|
954
1136
|
else if (retrievalMode === "semantic") {
|
|
955
|
-
const queryVec = await embedQuery(query);
|
|
956
|
-
|
|
957
|
-
|
|
1137
|
+
const queryVec = await embedQuery(query, signal);
|
|
1138
|
+
throwIfAborted(signal);
|
|
1139
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
1140
|
+
allResults.push(...vectorResults.results.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
958
1141
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
959
1142
|
vectorsByChunkId.set(chunkId, vector);
|
|
960
1143
|
}
|
|
@@ -964,13 +1147,14 @@ export class KnowledgeEngine {
|
|
|
964
1147
|
if (bm25Results.length === 0)
|
|
965
1148
|
continue;
|
|
966
1149
|
let vecResults = [];
|
|
967
|
-
const queryVec = await embedQuery(query);
|
|
968
|
-
|
|
1150
|
+
const queryVec = await embedQuery(query, signal);
|
|
1151
|
+
throwIfAborted(signal);
|
|
1152
|
+
const vectorResults = searchVectorFile(queryVec, vectorPath, iterateChunkIdsByKB(db, kb.id), candidateLimit, signal);
|
|
969
1153
|
vecResults = vectorResults.results;
|
|
970
1154
|
for (const [chunkId, vector] of vectorResults.vectorsByChunkId)
|
|
971
1155
|
vectorsByChunkId.set(chunkId, vector);
|
|
972
1156
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
973
|
-
allResults.push(...fused);
|
|
1157
|
+
allResults.push(...fused.map((result) => ({ ...result, score: result.score * kbTrustMultiplier(kb) })));
|
|
974
1158
|
}
|
|
975
1159
|
}
|
|
976
1160
|
// Deduplicate and sort
|
|
@@ -983,6 +1167,7 @@ export class KnowledgeEngine {
|
|
|
983
1167
|
});
|
|
984
1168
|
const rankingByChunkId = new Map();
|
|
985
1169
|
for (const result of unique) {
|
|
1170
|
+
throwIfAborted(signal);
|
|
986
1171
|
const chunk = getChunkById(db, result.chunkId);
|
|
987
1172
|
if (chunk) {
|
|
988
1173
|
const ranking = scoreChunkForQuery(result.score, chunk, queryTokens);
|
|
@@ -1030,9 +1215,11 @@ export class KnowledgeEngine {
|
|
|
1030
1215
|
return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
|
|
1031
1216
|
})
|
|
1032
1217
|
.filter(Boolean);
|
|
1033
|
-
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit));
|
|
1218
|
+
const reranked = await rerank(query, candidates, Math.max(limit * 3, limit), signal);
|
|
1219
|
+
throwIfAborted(signal);
|
|
1034
1220
|
const ranked = [];
|
|
1035
1221
|
for (const r of reranked) {
|
|
1222
|
+
throwIfAborted(signal);
|
|
1036
1223
|
const chunk = getChunkById(db, r.chunkId);
|
|
1037
1224
|
if (!chunk)
|
|
1038
1225
|
continue;
|
|
@@ -1051,17 +1238,30 @@ export class KnowledgeEngine {
|
|
|
1051
1238
|
}
|
|
1052
1239
|
const diversified = interleaveByFile(diversifyRankedChunks(ranked, diversity, vectorsByChunkId), diversity);
|
|
1053
1240
|
const page = diversified.slice(offset, offset + limit);
|
|
1054
|
-
const results = page.map((r) =>
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1241
|
+
const results = page.map((r) => {
|
|
1242
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1243
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1244
|
+
return {
|
|
1245
|
+
content: r.content,
|
|
1246
|
+
file_path: r.chunk.file_path,
|
|
1247
|
+
file_type: r.chunk.file_type,
|
|
1248
|
+
kb_name: r.kbName,
|
|
1249
|
+
score: r.score,
|
|
1250
|
+
ranking: r.ranking,
|
|
1251
|
+
snippet: r.snippet,
|
|
1252
|
+
start_line: r.startLine,
|
|
1253
|
+
end_line: r.endLine,
|
|
1254
|
+
provenance: {
|
|
1255
|
+
chunk_id: r.chunk.id,
|
|
1256
|
+
chunk_hash: r.chunk.content_hash,
|
|
1257
|
+
indexed_at: r.chunk.indexed_at,
|
|
1258
|
+
source_mtime: sourceMtime,
|
|
1259
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1260
|
+
match_reason: matchReasonFor(mode),
|
|
1261
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1262
|
+
},
|
|
1263
|
+
};
|
|
1264
|
+
});
|
|
1065
1265
|
return {
|
|
1066
1266
|
results,
|
|
1067
1267
|
total_count: diversified.length,
|
|
@@ -1072,6 +1272,7 @@ export class KnowledgeEngine {
|
|
|
1072
1272
|
}
|
|
1073
1273
|
const ranked = [];
|
|
1074
1274
|
for (const r of filtered) {
|
|
1275
|
+
throwIfAborted(signal);
|
|
1075
1276
|
const chunk = getChunkById(db, r.chunkId);
|
|
1076
1277
|
if (!chunk)
|
|
1077
1278
|
continue;
|
|
@@ -1118,6 +1319,19 @@ export class KnowledgeEngine {
|
|
|
1118
1319
|
snippet: r.snippet,
|
|
1119
1320
|
start_line: r.startLine,
|
|
1120
1321
|
end_line: r.endLine,
|
|
1322
|
+
provenance: (() => {
|
|
1323
|
+
const kb = kbById.get(r.chunk.kb_id);
|
|
1324
|
+
const sourceMtime = sourceMtimeFor(kb, r.chunk.file_path);
|
|
1325
|
+
return {
|
|
1326
|
+
chunk_id: r.chunk.id,
|
|
1327
|
+
chunk_hash: r.chunk.content_hash,
|
|
1328
|
+
indexed_at: r.chunk.indexed_at,
|
|
1329
|
+
source_mtime: sourceMtime,
|
|
1330
|
+
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1331
|
+
match_reason: matchReasonFor(mode),
|
|
1332
|
+
source_chunk_ids: r.sourceChunkIds,
|
|
1333
|
+
};
|
|
1334
|
+
})(),
|
|
1121
1335
|
}));
|
|
1122
1336
|
return {
|
|
1123
1337
|
results,
|
|
@@ -1136,31 +1350,94 @@ export class KnowledgeEngine {
|
|
|
1136
1350
|
remove(nameOrId) {
|
|
1137
1351
|
if (!this.db)
|
|
1138
1352
|
return false;
|
|
1353
|
+
if (this.disposing)
|
|
1354
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1355
|
+
if (this.activeMutations.size > 0)
|
|
1356
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1139
1357
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1140
1358
|
if (!kb)
|
|
1141
1359
|
return false;
|
|
1142
1360
|
deleteKB(this.db, kb.id);
|
|
1361
|
+
this.deleteVectorFile(kb.id);
|
|
1143
1362
|
return true;
|
|
1144
1363
|
}
|
|
1145
|
-
list() {
|
|
1364
|
+
list(signal) {
|
|
1146
1365
|
if (!this.db)
|
|
1147
1366
|
return [];
|
|
1367
|
+
throwIfAborted(signal);
|
|
1148
1368
|
return listKBs(this.db);
|
|
1149
1369
|
}
|
|
1150
1370
|
clear() {
|
|
1151
1371
|
if (!this.db)
|
|
1152
1372
|
return;
|
|
1153
|
-
|
|
1373
|
+
if (this.disposing)
|
|
1374
|
+
throw new Error("Knowledge engine is shutting down");
|
|
1375
|
+
if (this.activeMutations.size > 0)
|
|
1376
|
+
throw new Error("A knowledge-base mutation is already running");
|
|
1377
|
+
for (const kb of listKBs(this.db)) {
|
|
1154
1378
|
deleteKB(this.db, kb.id);
|
|
1379
|
+
this.deleteVectorFile(kb.id);
|
|
1380
|
+
}
|
|
1155
1381
|
}
|
|
1156
|
-
diagnose() {
|
|
1382
|
+
diagnose(signal) {
|
|
1157
1383
|
if (!this.db)
|
|
1158
1384
|
return [];
|
|
1385
|
+
throwIfAborted(signal);
|
|
1159
1386
|
const db = this.db;
|
|
1160
|
-
return listKBs(db).map((kb) =>
|
|
1387
|
+
return listKBs(db).map((kb) => {
|
|
1388
|
+
throwIfAborted(signal);
|
|
1389
|
+
return diagnoseKB(db, kb, signal);
|
|
1390
|
+
});
|
|
1161
1391
|
}
|
|
1162
|
-
|
|
1163
|
-
|
|
1392
|
+
symbolSearch(query, options = {}, signal) {
|
|
1393
|
+
if (!this.db)
|
|
1394
|
+
throw new Error("Engine not initialized");
|
|
1395
|
+
throwIfAborted(signal);
|
|
1396
|
+
const db = this.db;
|
|
1397
|
+
const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
|
|
1398
|
+
if (options.kb_id && !selectedKB)
|
|
1399
|
+
throw new Error(`Knowledge base not found: ${options.kb_id}`);
|
|
1400
|
+
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1401
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
1402
|
+
const symbolOptions = {
|
|
1403
|
+
kbId: selectedKB?.id,
|
|
1404
|
+
kind: options.kind,
|
|
1405
|
+
filePattern: options.file_pattern,
|
|
1406
|
+
exact: options.exact,
|
|
1407
|
+
};
|
|
1408
|
+
const symbols = searchSymbols(db, query, {
|
|
1409
|
+
...symbolOptions,
|
|
1410
|
+
limit: limit + 1,
|
|
1411
|
+
offset,
|
|
1412
|
+
});
|
|
1413
|
+
throwIfAborted(signal);
|
|
1414
|
+
const page = symbols.slice(0, limit);
|
|
1415
|
+
const total = countSymbols(db, query, symbolOptions);
|
|
1416
|
+
throwIfAborted(signal);
|
|
1417
|
+
return {
|
|
1418
|
+
results: page.map((symbol) => {
|
|
1419
|
+
const kb = getKB(db, symbol.kb_id);
|
|
1420
|
+
return {
|
|
1421
|
+
name: symbol.name,
|
|
1422
|
+
kind: symbol.kind,
|
|
1423
|
+
file_path: symbol.file_path,
|
|
1424
|
+
file_type: symbol.file_type,
|
|
1425
|
+
kb_name: kb?.name ?? "unknown",
|
|
1426
|
+
start_line: symbol.start_line,
|
|
1427
|
+
end_line: symbol.end_line,
|
|
1428
|
+
signature: symbol.signature ?? undefined,
|
|
1429
|
+
container_name: symbol.container_name ?? undefined,
|
|
1430
|
+
text: symbol.text,
|
|
1431
|
+
indexed_at: symbol.indexed_at,
|
|
1432
|
+
};
|
|
1433
|
+
}),
|
|
1434
|
+
total_count: total,
|
|
1435
|
+
has_more: symbols.length > limit,
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
doctor(signal) {
|
|
1439
|
+
throwIfAborted(signal);
|
|
1440
|
+
const diagnostics = this.diagnose(signal);
|
|
1164
1441
|
const issues = [];
|
|
1165
1442
|
const kbs = this.list();
|
|
1166
1443
|
if (kbs.length === 0) {
|
|
@@ -1168,9 +1445,11 @@ export class KnowledgeEngine {
|
|
|
1168
1445
|
severity: "blocking",
|
|
1169
1446
|
message: "No knowledge bases are indexed.",
|
|
1170
1447
|
action: "Run knowledge_add for the project root or relevant source/docs directory.",
|
|
1448
|
+
action_code: "rebuild_kb",
|
|
1171
1449
|
});
|
|
1172
1450
|
}
|
|
1173
1451
|
for (const diagnostic of diagnostics) {
|
|
1452
|
+
throwIfAborted(signal);
|
|
1174
1453
|
if (diagnostic.stuck_indexing) {
|
|
1175
1454
|
const phase = diagnostic.job ? ` during ${diagnostic.job.phase}` : "";
|
|
1176
1455
|
issues.push({
|
|
@@ -1178,6 +1457,7 @@ export class KnowledgeEngine {
|
|
|
1178
1457
|
kb_name: diagnostic.kb_name,
|
|
1179
1458
|
message: `Indexing appears stuck${phase} for ${formatDuration(diagnostic.last_progress_age_ms)}.`,
|
|
1180
1459
|
action: "Check knowledge_status for the last progress message. If no Pi process is actively indexing it, remove and rebuild this KB.",
|
|
1460
|
+
action_code: "wait_for_indexing",
|
|
1181
1461
|
});
|
|
1182
1462
|
}
|
|
1183
1463
|
if (diagnostic.status === "error") {
|
|
@@ -1186,6 +1466,7 @@ export class KnowledgeEngine {
|
|
|
1186
1466
|
kb_name: diagnostic.kb_name,
|
|
1187
1467
|
message: "Knowledge base is in error state and is skipped by search.",
|
|
1188
1468
|
action: "Run knowledge_remove and knowledge_add to rebuild it from the source.",
|
|
1469
|
+
action_code: "rebuild_kb",
|
|
1189
1470
|
});
|
|
1190
1471
|
}
|
|
1191
1472
|
if (diagnostic.coverage_percent < 80) {
|
|
@@ -1194,6 +1475,7 @@ export class KnowledgeEngine {
|
|
|
1194
1475
|
kb_name: diagnostic.kb_name,
|
|
1195
1476
|
message: `Coverage is ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files).`,
|
|
1196
1477
|
action: "Review skipped files and source path. Rebuild if index-time rules changed.",
|
|
1478
|
+
action_code: "review_skipped_scope",
|
|
1197
1479
|
});
|
|
1198
1480
|
}
|
|
1199
1481
|
if (diagnostic.stale_files.length > 0) {
|
|
@@ -1202,6 +1484,7 @@ export class KnowledgeEngine {
|
|
|
1202
1484
|
kb_name: diagnostic.kb_name,
|
|
1203
1485
|
message: `${diagnostic.stale_files.length} files changed after indexing.`,
|
|
1204
1486
|
action: "Run knowledge_update for this KB.",
|
|
1487
|
+
action_code: "run_update",
|
|
1205
1488
|
});
|
|
1206
1489
|
}
|
|
1207
1490
|
if (diagnostic.orphan_files.length > 0) {
|
|
@@ -1210,6 +1493,7 @@ export class KnowledgeEngine {
|
|
|
1210
1493
|
kb_name: diagnostic.kb_name,
|
|
1211
1494
|
message: `${diagnostic.orphan_files.length} indexed files no longer exist in the source.`,
|
|
1212
1495
|
action: "Run knowledge_update or rebuild the KB.",
|
|
1496
|
+
action_code: "run_update",
|
|
1213
1497
|
});
|
|
1214
1498
|
}
|
|
1215
1499
|
if (diagnostic.skipped_files.total > 0) {
|
|
@@ -1221,9 +1505,24 @@ export class KnowledgeEngine {
|
|
|
1221
1505
|
.map(([reason, count]) => `${reason}: ${count}`)
|
|
1222
1506
|
.join(", ")}).`,
|
|
1223
1507
|
action: "Use skipped samples to confirm exclusions are expected. Adjust source path or ignore rules if needed.",
|
|
1508
|
+
action_code: "review_skipped_scope",
|
|
1224
1509
|
});
|
|
1225
1510
|
}
|
|
1226
1511
|
}
|
|
1512
|
+
if (this.db) {
|
|
1513
|
+
for (const kb of kbs) {
|
|
1514
|
+
throwIfAborted(signal);
|
|
1515
|
+
if (kb.status === "ready" && kb.chunk_count > 0 && getSymbolCount(this.db, kb.id) === 0) {
|
|
1516
|
+
issues.push({
|
|
1517
|
+
severity: "info",
|
|
1518
|
+
kb_name: kb.name,
|
|
1519
|
+
message: "No lightweight symbols are indexed for this KB.",
|
|
1520
|
+
action: "Run knowledge_update to rebuild symbol/config/heading lookup metadata.",
|
|
1521
|
+
action_code: "run_update",
|
|
1522
|
+
});
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1227
1526
|
const penalty = issues.reduce((score, issue) => {
|
|
1228
1527
|
if (issue.severity === "blocking")
|
|
1229
1528
|
return score + 35;
|
|
@@ -1242,138 +1541,193 @@ export class KnowledgeEngine {
|
|
|
1242
1541
|
summary,
|
|
1243
1542
|
issues,
|
|
1244
1543
|
diagnostics,
|
|
1544
|
+
actions: issues.map((issue) => ({
|
|
1545
|
+
code: issue.action_code ?? "none",
|
|
1546
|
+
kb_name: issue.kb_name,
|
|
1547
|
+
target: issue.kb_name,
|
|
1548
|
+
description: issue.action,
|
|
1549
|
+
})),
|
|
1245
1550
|
};
|
|
1246
1551
|
}
|
|
1247
|
-
async exportKB(nameOrId, outputPath) {
|
|
1552
|
+
async exportKB(nameOrId, outputPath, signal, onProgress) {
|
|
1248
1553
|
if (!this.db)
|
|
1249
1554
|
throw new Error("Engine not initialized");
|
|
1555
|
+
throwIfAborted(signal);
|
|
1250
1556
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1251
1557
|
if (!kb)
|
|
1252
1558
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
1253
|
-
|
|
1254
|
-
|
|
1559
|
+
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
|
|
1560
|
+
}
|
|
1561
|
+
async exportKBUnlocked(kb, outputPath, signal, onProgress) {
|
|
1562
|
+
if (!this.db)
|
|
1563
|
+
throw new Error("Engine not initialized");
|
|
1564
|
+
const { createWriteStream } = await import("node:fs");
|
|
1565
|
+
const tempOutputPath = tempVectorPath(outputPath);
|
|
1566
|
+
const stream = createWriteStream(tempOutputPath, { encoding: "utf-8" });
|
|
1567
|
+
let count = 0;
|
|
1255
1568
|
const header = JSON.stringify({
|
|
1256
1569
|
name: kb.name,
|
|
1257
1570
|
description: kb.description,
|
|
1258
1571
|
source_type: "text",
|
|
1259
|
-
chunk_count:
|
|
1572
|
+
chunk_count: kb.chunk_count,
|
|
1260
1573
|
});
|
|
1261
|
-
|
|
1262
|
-
header
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1574
|
+
try {
|
|
1575
|
+
await writeLine(stream, header);
|
|
1576
|
+
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1577
|
+
throwIfAborted(signal);
|
|
1578
|
+
await writeLine(stream, JSON.stringify({
|
|
1579
|
+
content: chunk.content,
|
|
1580
|
+
file_path: chunk.file_path,
|
|
1581
|
+
file_type: chunk.file_type,
|
|
1582
|
+
start_line: chunk.start_line,
|
|
1583
|
+
end_line: chunk.end_line,
|
|
1584
|
+
metadata_json: chunk.metadata_json,
|
|
1585
|
+
}));
|
|
1586
|
+
count++;
|
|
1587
|
+
if (count % INDEX_EMBED_BATCH_SIZE === 0)
|
|
1588
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
|
|
1589
|
+
}
|
|
1590
|
+
await finishWriteStream(stream);
|
|
1591
|
+
throwIfAborted(signal);
|
|
1592
|
+
renameSync(tempOutputPath, outputPath);
|
|
1593
|
+
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
|
|
1594
|
+
return count;
|
|
1595
|
+
}
|
|
1596
|
+
catch (error) {
|
|
1597
|
+
stream.destroy();
|
|
1598
|
+
rmSync(tempOutputPath, { force: true });
|
|
1599
|
+
throw error;
|
|
1600
|
+
}
|
|
1274
1601
|
}
|
|
1275
1602
|
async importKB(inputPath, onProgress, signal) {
|
|
1276
1603
|
if (!this.db)
|
|
1277
1604
|
throw new Error("Engine not initialized");
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1605
|
+
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
|
|
1606
|
+
}
|
|
1607
|
+
async importKBUnlocked(inputPath, onProgress, signal) {
|
|
1608
|
+
if (!this.db)
|
|
1609
|
+
throw new Error("Engine not initialized");
|
|
1610
|
+
throwIfAborted(signal);
|
|
1611
|
+
const { createReadStream } = await import("node:fs");
|
|
1612
|
+
const { createInterface } = await import("node:readline");
|
|
1613
|
+
const stream = createReadStream(inputPath, { encoding: "utf-8" });
|
|
1614
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
1615
|
+
let header;
|
|
1616
|
+
let kb;
|
|
1286
1617
|
let vectorWriter;
|
|
1287
1618
|
let tempVectorFile;
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1619
|
+
let inserted = 0;
|
|
1620
|
+
let lineNumber = 0;
|
|
1621
|
+
const pendingChunks = [];
|
|
1622
|
+
const pendingSymbols = [];
|
|
1623
|
+
const flushPending = async () => {
|
|
1624
|
+
if (!this.db || !kb || !vectorWriter || pendingChunks.length === 0)
|
|
1625
|
+
return;
|
|
1626
|
+
throwIfAborted(signal);
|
|
1627
|
+
const batch = pendingChunks.splice(0, INDEX_EMBED_BATCH_SIZE);
|
|
1628
|
+
const symbols = pendingSymbols.splice(0);
|
|
1629
|
+
const total = header?.chunk_count;
|
|
1630
|
+
const message = `Embedding import batch: ${inserted}/${total ?? "unknown"} chunks stored`;
|
|
1291
1631
|
updateIndexingJob(this.db, kb.id, {
|
|
1292
|
-
phase: "
|
|
1293
|
-
message
|
|
1294
|
-
|
|
1632
|
+
phase: "embedding",
|
|
1633
|
+
message,
|
|
1634
|
+
processed_chunks: inserted,
|
|
1635
|
+
total_files: total,
|
|
1636
|
+
added_chunks: inserted,
|
|
1295
1637
|
});
|
|
1296
|
-
onProgress?.(
|
|
1297
|
-
const
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1638
|
+
onProgress?.(message);
|
|
1639
|
+
const vectors = await embedDocuments(batch.map((chunk) => buildChunkEmbeddingText(chunk)), signal);
|
|
1640
|
+
throwIfAborted(signal);
|
|
1641
|
+
insertChunks(this.db, kb.id, batch);
|
|
1642
|
+
insertSymbols(this.db, kb.id, symbols);
|
|
1643
|
+
vectorWriter.append(vectors);
|
|
1644
|
+
inserted += batch.length;
|
|
1645
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1646
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1647
|
+
phase: "storing",
|
|
1648
|
+
message: `Stored import batch: ${inserted}/${total ?? "unknown"} chunks`,
|
|
1649
|
+
processed_chunks: inserted,
|
|
1650
|
+
total_files: total,
|
|
1651
|
+
added_chunks: inserted,
|
|
1652
|
+
});
|
|
1653
|
+
};
|
|
1654
|
+
try {
|
|
1655
|
+
for await (const rawLine of lines) {
|
|
1656
|
+
throwIfAborted(signal);
|
|
1657
|
+
lineNumber++;
|
|
1658
|
+
const line = rawLine.trim();
|
|
1659
|
+
if (!line)
|
|
1660
|
+
continue;
|
|
1661
|
+
if (!header) {
|
|
1662
|
+
header = JSON.parse(line);
|
|
1663
|
+
if (!header.name)
|
|
1664
|
+
throw new Error("Import header is missing a knowledge base name");
|
|
1665
|
+
const existingKB = getKBByName(this.db, header.name);
|
|
1666
|
+
if (existingKB) {
|
|
1667
|
+
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
1668
|
+
}
|
|
1669
|
+
kb = createKB(this.db, { name: header.name, description: header.description, source_type: "text" });
|
|
1670
|
+
updateKBStatus(this.db, kb.id, "indexing");
|
|
1671
|
+
startIndexingJob(this.db, kb.id, "import", `Starting import for "${header.name}"`);
|
|
1672
|
+
const importMessage = `Importing ${header.chunk_count ?? "unknown"} chunks...`;
|
|
1673
|
+
updateIndexingJob(this.db, kb.id, {
|
|
1674
|
+
phase: "importing",
|
|
1675
|
+
message: importMessage,
|
|
1676
|
+
total_files: header.chunk_count,
|
|
1677
|
+
});
|
|
1678
|
+
onProgress?.(importMessage);
|
|
1679
|
+
const vectorPath = this.vectorPathFor(kb.id);
|
|
1680
|
+
tempVectorFile = tempVectorPath(vectorPath);
|
|
1681
|
+
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1682
|
+
continue;
|
|
1683
|
+
}
|
|
1684
|
+
const imported = JSON.parse(line);
|
|
1685
|
+
const chunk = importedChunkToInsert(imported);
|
|
1686
|
+
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
1687
|
+
pendingChunks.push(chunk);
|
|
1688
|
+
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
1689
|
+
if (pendingChunks.length >= INDEX_EMBED_BATCH_SIZE)
|
|
1690
|
+
await flushPending();
|
|
1349
1691
|
}
|
|
1692
|
+
if (!header)
|
|
1693
|
+
throw new Error("Empty import file");
|
|
1694
|
+
if (!kb || !vectorWriter || !tempVectorFile)
|
|
1695
|
+
throw new Error("Import did not create a knowledge base");
|
|
1696
|
+
await flushPending();
|
|
1350
1697
|
vectorWriter.close();
|
|
1351
1698
|
vectorWriter = undefined;
|
|
1352
|
-
renameSync(tempVectorFile,
|
|
1699
|
+
renameSync(tempVectorFile, this.vectorPathFor(kb.id));
|
|
1353
1700
|
tempVectorFile = undefined;
|
|
1354
|
-
updateKBCounts(this.db, kb.id,
|
|
1701
|
+
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
1355
1702
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1356
1703
|
const savedKB = getKB(this.db, kb.id);
|
|
1357
1704
|
if (!savedKB)
|
|
1358
1705
|
throw new Error(`Knowledge base disappeared after import: ${kb.id}`);
|
|
1359
|
-
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${
|
|
1360
|
-
return { kb: savedKB, chunkCount:
|
|
1706
|
+
finishIndexingJob(this.db, kb.id, "succeeded", `Ready: imported ${inserted} chunks`);
|
|
1707
|
+
return { kb: savedKB, chunkCount: inserted };
|
|
1361
1708
|
}
|
|
1362
|
-
catch (
|
|
1709
|
+
catch (error) {
|
|
1710
|
+
lines.close();
|
|
1711
|
+
stream.destroy();
|
|
1363
1712
|
vectorWriter?.close();
|
|
1364
1713
|
if (tempVectorFile)
|
|
1365
1714
|
rmSync(tempVectorFile, { force: true });
|
|
1366
|
-
if (this.db) {
|
|
1367
|
-
finishIndexingJob(this.db, kb.id, isCancellationError(
|
|
1715
|
+
if (this.db && kb) {
|
|
1716
|
+
finishIndexingJob(this.db, kb.id, isCancellationError(error) ? "cancelled" : "failed", isCancellationError(error) ? "Import cancelled." : `Import failed near line ${lineNumber}.`, error instanceof Error ? error.message : String(error));
|
|
1717
|
+
deleteKB(this.db, kb.id);
|
|
1718
|
+
this.deleteVectorFile(kb.id);
|
|
1368
1719
|
}
|
|
1369
|
-
|
|
1370
|
-
throw e;
|
|
1720
|
+
throw error;
|
|
1371
1721
|
}
|
|
1372
1722
|
}
|
|
1373
1723
|
async dispose(options = {}) {
|
|
1724
|
+
this.disposing = true;
|
|
1374
1725
|
const activeUpdates = [...this.activeUpdates.values()];
|
|
1375
1726
|
if (activeUpdates.length > 0)
|
|
1376
1727
|
await Promise.allSettled(activeUpdates);
|
|
1728
|
+
const activeMutations = [...this.activeMutations.values()];
|
|
1729
|
+
if (activeMutations.length > 0)
|
|
1730
|
+
await Promise.allSettled(activeMutations);
|
|
1377
1731
|
const disposeModels = options.disposeModels ?? true;
|
|
1378
1732
|
if (disposeModels) {
|
|
1379
1733
|
await disposeEmbedding();
|