pi-knowledge 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +59 -22
- package/dist/index.d.ts +10 -4
- package/dist/index.js +145 -166
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +13 -8
- package/dist/src/engine.js +569 -100
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +10 -0
- package/dist/src/indexer/chunker.js +284 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +184 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +47 -0
- package/docs/configuration.md +39 -9
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
package/dist/src/engine.js
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
import { existsSync,
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { closeSync, constants, createReadStream, createWriteStream, existsSync, fstatSync, fsyncSync, lstatSync, openSync, renameSync, rmSync, statSync, } from "node:fs";
|
|
2
|
+
import { lstat as lstatFile, mkdtemp, open as openFile, realpath as realpathFile, rm as rmAsync, writeFile, } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
|
+
import { createInterface } from "node:readline";
|
|
6
|
+
import { diagnoseKB, diagnoseKBAsync } from "./diagnostics/health.js";
|
|
4
7
|
import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
8
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
|
-
import {
|
|
9
|
+
import { assertNetworkAccess, assertOpenedPathAuthorized, assertOutputPathAuthorized, authorizeExistingPath, authorizeOutputPath, } from "./filesystem-access.js";
|
|
10
|
+
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, iterateScannableFilesAsync, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
11
|
import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
|
|
8
12
|
import { htmlToStructuredMarkdown } from "./indexer/html.js";
|
|
9
13
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
10
14
|
import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
|
|
15
|
+
import { fetchPublicText } from "./network/safe-fetch.js";
|
|
11
16
|
import { searchBM25 } from "./search/bm25.js";
|
|
12
17
|
import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
|
|
13
18
|
import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
|
|
@@ -17,7 +22,8 @@ import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilte
|
|
|
17
22
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
18
23
|
import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
|
|
19
24
|
import { searchVectorFile } from "./search/vector.js";
|
|
20
|
-
import {
|
|
25
|
+
import { ensureOwnerOnlyDirectory, prepareSafeNewFile, publishNewFile } from "./secure-files.js";
|
|
26
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, getSymbolsByFile, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, markKBIndexed, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
|
|
21
27
|
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
|
|
22
28
|
export const CURRENT_URL_STRUCTURE_POLICY = 1;
|
|
23
29
|
export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
|
|
@@ -63,11 +69,12 @@ function matchReasonFor(mode) {
|
|
|
63
69
|
return "bm25";
|
|
64
70
|
return "hybrid";
|
|
65
71
|
}
|
|
66
|
-
function toScanOptions(options = {}) {
|
|
72
|
+
function toScanOptions(options = {}, accessPolicy) {
|
|
67
73
|
return {
|
|
68
74
|
includeSuggestedText: options.include_suggested_text === true,
|
|
69
75
|
includePaths: options.include_paths,
|
|
70
76
|
excludePaths: options.exclude_paths,
|
|
77
|
+
accessPolicy,
|
|
71
78
|
};
|
|
72
79
|
}
|
|
73
80
|
function serializeAddOptions(options = {}) {
|
|
@@ -131,16 +138,22 @@ function planDirectoryScan(dirPath, options = {}, signal) {
|
|
|
131
138
|
skipped,
|
|
132
139
|
};
|
|
133
140
|
}
|
|
134
|
-
function
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
return undefined;
|
|
141
|
+
async function planDirectoryScanAsync(dirPath, options = {}, signal) {
|
|
142
|
+
const skipped = createSkippedScanStats();
|
|
143
|
+
let files = 0;
|
|
144
|
+
let bytes = 0;
|
|
145
|
+
for await (const file of iterateScannableFilesAsync(dirPath, skipped, options, signal)) {
|
|
146
|
+
throwIfAborted(signal);
|
|
147
|
+
files++;
|
|
148
|
+
bytes += file.size;
|
|
143
149
|
}
|
|
150
|
+
return {
|
|
151
|
+
files,
|
|
152
|
+
bytes,
|
|
153
|
+
skippedTotal: skipped.total,
|
|
154
|
+
skippedSummary: summarizeSkippedScan(skipped),
|
|
155
|
+
skipped,
|
|
156
|
+
};
|
|
144
157
|
}
|
|
145
158
|
function kbTrustMultiplier(kb) {
|
|
146
159
|
let multiplier = kb.status === "stale" ? 0.92 : 1;
|
|
@@ -535,35 +548,146 @@ function isWeakAutoResponse(query, response, primaryMode, attemptMode) {
|
|
|
535
548
|
}
|
|
536
549
|
return false;
|
|
537
550
|
}
|
|
551
|
+
const MAX_DIRECT_TEXT_BYTES = 10 * 1024 * 1024;
|
|
552
|
+
const MAX_DIRECT_DOCUMENT_BYTES = 64 * 1024 * 1024;
|
|
553
|
+
const MAX_IMPORT_BYTES = 256 * 1024 * 1024;
|
|
554
|
+
const MAX_IMPORT_LINE_BYTES = 16 * 1024 * 1024;
|
|
555
|
+
const MAX_IMPORT_LINES = 1_000_001;
|
|
556
|
+
const MAX_IMPORT_CHUNKS = 1_000_000;
|
|
557
|
+
const MAX_EXPORT_BYTES = MAX_IMPORT_BYTES;
|
|
558
|
+
const MAX_EXPORT_CHUNKS = MAX_IMPORT_CHUNKS;
|
|
559
|
+
const MAX_EXPORT_DURATION_MS = 60 * 60_000;
|
|
560
|
+
const MAX_KB_NAME_BYTES = 512;
|
|
561
|
+
const MAX_KB_DESCRIPTION_BYTES = 4 * 1024;
|
|
562
|
+
const MAX_IMPORT_CONTENT_BYTES = 12 * 1024 * 1024;
|
|
563
|
+
const MAX_IMPORT_FILE_PATH_BYTES = 4_096;
|
|
564
|
+
const MAX_IMPORT_FILE_TYPE_BYTES = 256;
|
|
565
|
+
const MAX_IMPORT_METADATA_BYTES = 64 * 1024;
|
|
566
|
+
const MAX_IMPORT_METADATA_CONTEXT_BYTES = 8 * 1024;
|
|
567
|
+
const MAX_IMPORT_METADATA_SCOPE_ITEMS = 128;
|
|
568
|
+
const MAX_IMPORT_LINE_NUMBER = 2_147_483_647;
|
|
569
|
+
const MAX_IMPORT_FIXED_CONTEXT_BYTES = 64 * 1024;
|
|
570
|
+
const IMPORTED_CHUNK_KEYS = new Set(["content", "file_path", "file_type", "start_line", "end_line", "metadata_json"]);
|
|
571
|
+
const IMPORTED_CONTEXT_STRING_KEYS = [
|
|
572
|
+
"heading",
|
|
573
|
+
"breadcrumb",
|
|
574
|
+
"language",
|
|
575
|
+
"symbol",
|
|
576
|
+
"function_name",
|
|
577
|
+
"symbol_kind",
|
|
578
|
+
"parent_symbol",
|
|
579
|
+
"signature",
|
|
580
|
+
];
|
|
538
581
|
async function chunkUrl(source, signal) {
|
|
539
|
-
const
|
|
540
|
-
if (!res.ok)
|
|
541
|
-
throw new Error(`Fetch failed: ${res.status}`);
|
|
542
|
-
const html = await res.text();
|
|
582
|
+
const html = await fetchPublicText(source, signal);
|
|
543
583
|
throwIfAborted(signal);
|
|
544
584
|
const text = htmlToStructuredMarkdown(html);
|
|
545
585
|
const structuredChunks = chunkMarkdown(text, source, "html");
|
|
546
586
|
return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
|
|
547
587
|
}
|
|
548
|
-
|
|
588
|
+
function validateKnowledgeBaseMetadata(name, description) {
|
|
589
|
+
if (typeof name !== "string" || !name.trim())
|
|
590
|
+
throw new Error("Knowledge base name must be a non-empty string");
|
|
591
|
+
if (Buffer.byteLength(name) > MAX_KB_NAME_BYTES) {
|
|
592
|
+
throw new Error(`Knowledge base name exceeds the ${MAX_KB_NAME_BYTES}-byte limit`);
|
|
593
|
+
}
|
|
594
|
+
const normalizedDescription = description === null ? undefined : description;
|
|
595
|
+
if (normalizedDescription !== undefined && typeof normalizedDescription !== "string") {
|
|
596
|
+
throw new Error("Knowledge base description must be a string");
|
|
597
|
+
}
|
|
598
|
+
if (typeof normalizedDescription === "string" &&
|
|
599
|
+
Buffer.byteLength(normalizedDescription) > MAX_KB_DESCRIPTION_BYTES) {
|
|
600
|
+
throw new Error(`Knowledge base description exceeds the ${MAX_KB_DESCRIPTION_BYTES}-byte limit`);
|
|
601
|
+
}
|
|
602
|
+
return { name, description: normalizedDescription };
|
|
603
|
+
}
|
|
604
|
+
async function readBoundedRegularFile(filePath, maxBytes, signal, expected, access) {
|
|
605
|
+
if (expected?.rootPath) {
|
|
606
|
+
const canonical = await realpathFile(filePath);
|
|
607
|
+
const rel = relative(expected.rootPath, canonical);
|
|
608
|
+
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
|
|
609
|
+
throw new Error(`Source file escaped its scanned root: ${filePath}`);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
const direct = lstatSync(filePath);
|
|
613
|
+
if (direct.isSymbolicLink() || !direct.isFile())
|
|
614
|
+
throw new Error(`Source is not a regular non-symlink file: ${filePath}`);
|
|
615
|
+
if (expected?.device !== undefined &&
|
|
616
|
+
expected.inode !== undefined &&
|
|
617
|
+
(direct.dev !== expected.device || direct.ino !== expected.inode)) {
|
|
618
|
+
throw new Error(`Source file changed after scanning: ${filePath}`);
|
|
619
|
+
}
|
|
620
|
+
const handle = await openFile(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
621
|
+
try {
|
|
622
|
+
const opened = await handle.stat();
|
|
623
|
+
if (!opened.isFile())
|
|
624
|
+
throw new Error(`Source is not a regular file: ${filePath}`);
|
|
625
|
+
assertOpenedPathAuthorized(filePath, opened.dev, opened.ino, access);
|
|
626
|
+
if (expected?.rootPath) {
|
|
627
|
+
const canonical = await realpathFile(filePath);
|
|
628
|
+
const current = await lstatFile(filePath);
|
|
629
|
+
const rel = relative(expected.rootPath, canonical);
|
|
630
|
+
if (rel === ".." ||
|
|
631
|
+
rel.startsWith(`..${sep}`) ||
|
|
632
|
+
isAbsolute(rel) ||
|
|
633
|
+
current.isSymbolicLink() ||
|
|
634
|
+
current.dev !== opened.dev ||
|
|
635
|
+
current.ino !== opened.ino) {
|
|
636
|
+
throw new Error(`Source file changed outside its scanned root: ${filePath}`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (expected?.device !== undefined &&
|
|
640
|
+
expected.inode !== undefined &&
|
|
641
|
+
(opened.dev !== expected.device || opened.ino !== expected.inode)) {
|
|
642
|
+
throw new Error(`Source file changed while opening: ${filePath}`);
|
|
643
|
+
}
|
|
644
|
+
if (opened.size > maxBytes) {
|
|
645
|
+
throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
|
|
646
|
+
}
|
|
647
|
+
const chunks = [];
|
|
648
|
+
let bytes = 0;
|
|
649
|
+
while (bytes <= maxBytes) {
|
|
650
|
+
throwIfAborted(signal);
|
|
651
|
+
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes - bytes + 1));
|
|
652
|
+
const result = await handle.read(buffer, 0, buffer.length, null);
|
|
653
|
+
if (result.bytesRead === 0)
|
|
654
|
+
break;
|
|
655
|
+
bytes += result.bytesRead;
|
|
656
|
+
if (bytes > maxBytes) {
|
|
657
|
+
throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
|
|
658
|
+
}
|
|
659
|
+
chunks.push(buffer.subarray(0, result.bytesRead));
|
|
660
|
+
}
|
|
661
|
+
return Buffer.concat(chunks, bytes);
|
|
662
|
+
}
|
|
663
|
+
finally {
|
|
664
|
+
await handle.close();
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
async function extractSourceFileContent(filePath, signal, expected, access) {
|
|
668
|
+
filePath = authorizeExistingPath(filePath, access);
|
|
549
669
|
const lowerPath = filePath.toLowerCase();
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
670
|
+
const isDocument = lowerPath.endsWith(".pdf") || lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc");
|
|
671
|
+
const maxBytes = isDocument ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
|
|
672
|
+
const content = await readBoundedRegularFile(filePath, maxBytes, signal, expected, access);
|
|
673
|
+
if (!isDocument) {
|
|
674
|
+
if (content.subarray(0, 8_000).includes(0x00)) {
|
|
675
|
+
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
676
|
+
}
|
|
677
|
+
return { content: content.toString("utf8"), fileType: "text" };
|
|
555
678
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
679
|
+
const tempDirectory = ensureOwnerOnlyDirectory(await mkdtemp(join(tmpdir(), "pi-knowledge-document-")));
|
|
680
|
+
const tempPath = join(tempDirectory, "source");
|
|
681
|
+
try {
|
|
682
|
+
await writeFile(tempPath, content, { flag: "wx", mode: 0o600 });
|
|
683
|
+
const fileType = lowerPath.endsWith(".pdf") ? "pdf" : "docx";
|
|
684
|
+
const extracted = await extractDocumentInModelWorker(tempPath, fileType, signal);
|
|
559
685
|
throwIfAborted(signal);
|
|
560
686
|
return extracted;
|
|
561
687
|
}
|
|
562
|
-
|
|
563
|
-
|
|
688
|
+
finally {
|
|
689
|
+
await rmAsync(tempDirectory, { recursive: true, force: true });
|
|
564
690
|
}
|
|
565
|
-
throwIfAborted(signal);
|
|
566
|
-
return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
|
|
567
691
|
}
|
|
568
692
|
function looksLikeLocalPath(source) {
|
|
569
693
|
const trimmed = source.trim();
|
|
@@ -577,11 +701,15 @@ function looksLikeLocalPath(source) {
|
|
|
577
701
|
return true;
|
|
578
702
|
return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
|
|
579
703
|
}
|
|
580
|
-
function classifySource(source) {
|
|
581
|
-
const resolvedSource = resolve(source);
|
|
704
|
+
function classifySource(source, access) {
|
|
582
705
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
583
|
-
|
|
584
|
-
|
|
706
|
+
if (isUrl)
|
|
707
|
+
assertNetworkAccess(access);
|
|
708
|
+
const requestedSource = resolve(source);
|
|
709
|
+
const exists = !isUrl && existsSync(requestedSource);
|
|
710
|
+
const resolvedSource = exists ? authorizeExistingPath(requestedSource, access) : requestedSource;
|
|
711
|
+
const isDir = exists && statSync(resolvedSource).isDirectory();
|
|
712
|
+
const isFile = exists && statSync(resolvedSource).isFile();
|
|
585
713
|
if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
|
|
586
714
|
throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
|
|
587
715
|
}
|
|
@@ -594,7 +722,7 @@ function classifySource(source) {
|
|
|
594
722
|
};
|
|
595
723
|
}
|
|
596
724
|
async function extractScannableFileContent(file, signal) {
|
|
597
|
-
return extractSourceFileContent(file.path, signal);
|
|
725
|
+
return extractSourceFileContent(file.path, signal, file);
|
|
598
726
|
}
|
|
599
727
|
async function extractScannableFileContentOrSkip(file, skipped, signal) {
|
|
600
728
|
try {
|
|
@@ -622,6 +750,100 @@ function throwIfAborted(signal) {
|
|
|
622
750
|
if (signal?.aborted)
|
|
623
751
|
throw new Error("Cancelled");
|
|
624
752
|
}
|
|
753
|
+
function boundedImportString(value, field, maxBytes) {
|
|
754
|
+
if (typeof value !== "string")
|
|
755
|
+
throw new Error(`Imported chunk ${field} must be a string`);
|
|
756
|
+
if (Buffer.byteLength(value) > maxBytes) {
|
|
757
|
+
throw new Error(`Imported chunk ${field} exceeds the ${maxBytes}-byte limit`);
|
|
758
|
+
}
|
|
759
|
+
return value;
|
|
760
|
+
}
|
|
761
|
+
function validateImportedMetadata(metadataJson) {
|
|
762
|
+
let parsed;
|
|
763
|
+
try {
|
|
764
|
+
parsed = JSON.parse(metadataJson);
|
|
765
|
+
}
|
|
766
|
+
catch {
|
|
767
|
+
throw new Error("Imported chunk metadata_json must contain valid JSON");
|
|
768
|
+
}
|
|
769
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
770
|
+
throw new Error("Imported chunk metadata_json must contain an object");
|
|
771
|
+
}
|
|
772
|
+
const metadata = parsed;
|
|
773
|
+
let contextBytes = 0;
|
|
774
|
+
for (const key of IMPORTED_CONTEXT_STRING_KEYS) {
|
|
775
|
+
const value = metadata[key];
|
|
776
|
+
if (value === undefined)
|
|
777
|
+
continue;
|
|
778
|
+
if (typeof value !== "string")
|
|
779
|
+
throw new Error(`Imported chunk metadata ${key} must be a string`);
|
|
780
|
+
contextBytes += Buffer.byteLength(value);
|
|
781
|
+
}
|
|
782
|
+
const scope = metadata.scope;
|
|
783
|
+
if (scope !== undefined) {
|
|
784
|
+
if (!Array.isArray(scope) || scope.length > MAX_IMPORT_METADATA_SCOPE_ITEMS) {
|
|
785
|
+
throw new Error("Imported chunk metadata scope is invalid");
|
|
786
|
+
}
|
|
787
|
+
for (const item of scope) {
|
|
788
|
+
if (typeof item !== "string")
|
|
789
|
+
throw new Error("Imported chunk metadata scope must contain strings");
|
|
790
|
+
contextBytes += Buffer.byteLength(item);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
if (contextBytes > MAX_IMPORT_METADATA_CONTEXT_BYTES) {
|
|
794
|
+
throw new Error(`Imported chunk embedding metadata exceeds the ${MAX_IMPORT_METADATA_CONTEXT_BYTES}-byte limit`);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
function parseImportedChunk(value) {
|
|
798
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
799
|
+
throw new Error("Imported chunk must be an object");
|
|
800
|
+
}
|
|
801
|
+
const record = value;
|
|
802
|
+
for (const key of Object.keys(record)) {
|
|
803
|
+
if (!IMPORTED_CHUNK_KEYS.has(key))
|
|
804
|
+
throw new Error(`Imported chunk contains unknown field: ${key}`);
|
|
805
|
+
}
|
|
806
|
+
const content = boundedImportString(record.content, "content", MAX_IMPORT_CONTENT_BYTES);
|
|
807
|
+
const filePath = boundedImportString(record.file_path, "file_path", MAX_IMPORT_FILE_PATH_BYTES);
|
|
808
|
+
const fileType = boundedImportString(record.file_type, "file_type", MAX_IMPORT_FILE_TYPE_BYTES);
|
|
809
|
+
if (!content)
|
|
810
|
+
throw new Error("Imported chunk content must not be empty");
|
|
811
|
+
if (!filePath)
|
|
812
|
+
throw new Error("Imported chunk file_path must not be empty");
|
|
813
|
+
if (!fileType)
|
|
814
|
+
throw new Error("Imported chunk file_type must not be empty");
|
|
815
|
+
const startLine = record.start_line;
|
|
816
|
+
const endLine = record.end_line;
|
|
817
|
+
if (!Number.isSafeInteger(startLine) ||
|
|
818
|
+
typeof startLine !== "number" ||
|
|
819
|
+
startLine < 0 ||
|
|
820
|
+
startLine > MAX_IMPORT_LINE_NUMBER) {
|
|
821
|
+
throw new Error("Imported chunk start_line is invalid");
|
|
822
|
+
}
|
|
823
|
+
if (!Number.isSafeInteger(endLine) ||
|
|
824
|
+
typeof endLine !== "number" ||
|
|
825
|
+
endLine < startLine ||
|
|
826
|
+
endLine > MAX_IMPORT_LINE_NUMBER) {
|
|
827
|
+
throw new Error("Imported chunk end_line is invalid");
|
|
828
|
+
}
|
|
829
|
+
const metadataJson = record.metadata_json === undefined
|
|
830
|
+
? "{}"
|
|
831
|
+
: boundedImportString(record.metadata_json, "metadata_json", MAX_IMPORT_METADATA_BYTES);
|
|
832
|
+
validateImportedMetadata(metadataJson);
|
|
833
|
+
const imported = {
|
|
834
|
+
content,
|
|
835
|
+
file_path: filePath,
|
|
836
|
+
file_type: fileType,
|
|
837
|
+
start_line: startLine,
|
|
838
|
+
end_line: endLine,
|
|
839
|
+
metadata_json: metadataJson,
|
|
840
|
+
};
|
|
841
|
+
const fixedContext = buildChunkEmbeddingText({ ...imported, content: "" });
|
|
842
|
+
if (Buffer.byteLength(fixedContext) > MAX_IMPORT_FIXED_CONTEXT_BYTES) {
|
|
843
|
+
throw new Error(`Imported chunk fixed embedding context exceeds the ${MAX_IMPORT_FIXED_CONTEXT_BYTES}-byte limit`);
|
|
844
|
+
}
|
|
845
|
+
return imported;
|
|
846
|
+
}
|
|
625
847
|
function importedChunkToInsert(chunk) {
|
|
626
848
|
const metadataJson = chunk.metadata_json || "{}";
|
|
627
849
|
return {
|
|
@@ -718,9 +940,12 @@ export class KnowledgeEngine {
|
|
|
718
940
|
deleteVectorFile(kbId) {
|
|
719
941
|
rmSync(this.vectorPathFor(kbId), { force: true });
|
|
720
942
|
}
|
|
721
|
-
plan(source, options = {}, signal) {
|
|
943
|
+
plan(source, options = {}, signal, access) {
|
|
722
944
|
throwIfAborted(signal);
|
|
723
|
-
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
|
|
945
|
+
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
|
|
946
|
+
if ((isDir || isFile) && lstatSync(resolvedSource).isSymbolicLink()) {
|
|
947
|
+
throw new Error(`Source is not a regular non-symlink path: ${resolvedSource}`);
|
|
948
|
+
}
|
|
724
949
|
if (isDir) {
|
|
725
950
|
const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
|
|
726
951
|
return {
|
|
@@ -764,17 +989,53 @@ export class KnowledgeEngine {
|
|
|
764
989
|
summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
|
|
765
990
|
};
|
|
766
991
|
}
|
|
767
|
-
async
|
|
992
|
+
async planAsync(source, options = {}, signal, access) {
|
|
993
|
+
throwIfAborted(signal);
|
|
994
|
+
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
|
|
995
|
+
if (isDir) {
|
|
996
|
+
const plan = await planDirectoryScanAsync(resolvedSource, toScanOptions(options, access), signal);
|
|
997
|
+
return {
|
|
998
|
+
source_type: "directory",
|
|
999
|
+
scannable_files: plan.files,
|
|
1000
|
+
scannable_bytes: plan.bytes,
|
|
1001
|
+
skipped: plan.skipped,
|
|
1002
|
+
summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
if (isFile && lstatSync(resolvedSource).isSymbolicLink()) {
|
|
1006
|
+
throw new Error(`Source is not a regular non-symlink file: ${resolvedSource}`);
|
|
1007
|
+
}
|
|
1008
|
+
const skipped = createSkippedScanStats();
|
|
1009
|
+
const sourceBytes = isFile ? statSync(resolvedSource).size : Buffer.byteLength(source);
|
|
1010
|
+
const maxBytes = isFile && isSupportedDocumentFile(resolvedSource) ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
|
|
1011
|
+
if (sourceType !== "url" && sourceBytes > maxBytes) {
|
|
1012
|
+
addSkippedScanEntry(skipped, {
|
|
1013
|
+
path: isFile ? resolvedSource : "<inline>",
|
|
1014
|
+
reason: "oversized",
|
|
1015
|
+
size: sourceBytes,
|
|
1016
|
+
});
|
|
1017
|
+
return {
|
|
1018
|
+
source_type: sourceType,
|
|
1019
|
+
scannable_files: 0,
|
|
1020
|
+
scannable_bytes: 0,
|
|
1021
|
+
skipped,
|
|
1022
|
+
summary: `${isFile ? "File" : "Inline text"} plan: 0 scannable sources; source exceeds ${formatBytes(maxBytes)}`,
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
return this.plan(source, options, signal, access);
|
|
1026
|
+
}
|
|
1027
|
+
async add(source, name, onProgress, signal, options = {}, access) {
|
|
768
1028
|
throwIfAborted(signal);
|
|
769
|
-
|
|
1029
|
+
validateKnowledgeBaseMetadata(name, undefined);
|
|
1030
|
+
return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options, access));
|
|
770
1031
|
}
|
|
771
|
-
async addUnlocked(source, name, onProgress, signal, options = {}) {
|
|
1032
|
+
async addUnlocked(source, name, onProgress, signal, options = {}, access) {
|
|
772
1033
|
if (!this.db)
|
|
773
1034
|
throw new Error("Engine not initialized");
|
|
774
1035
|
throwIfAborted(signal);
|
|
775
1036
|
const db = this.db;
|
|
776
|
-
const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
|
|
777
|
-
const scanOptions = toScanOptions(options);
|
|
1037
|
+
const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source, access);
|
|
1038
|
+
const scanOptions = toScanOptions(options, access);
|
|
778
1039
|
const existingKB = getKBByName(db, name);
|
|
779
1040
|
if (existingKB) {
|
|
780
1041
|
throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
|
|
@@ -874,13 +1135,13 @@ export class KnowledgeEngine {
|
|
|
874
1135
|
await addChunks(chunks);
|
|
875
1136
|
}
|
|
876
1137
|
else if (isFile) {
|
|
877
|
-
const extracted = await extractSourceFileContent(resolvedSource, signal);
|
|
1138
|
+
const extracted = await extractSourceFileContent(resolvedSource, signal, undefined, access);
|
|
878
1139
|
fileCount = 1;
|
|
879
1140
|
const chunks = await analyzeAndAddSymbols(extracted.content, resolvedSource, extracted.fileType);
|
|
880
1141
|
await addChunks(chunks);
|
|
881
1142
|
}
|
|
882
1143
|
else if (isDir) {
|
|
883
|
-
const plan =
|
|
1144
|
+
const plan = await planDirectoryScanAsync(resolvedSource, scanOptions, signal);
|
|
884
1145
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
885
1146
|
updateIndexingJob(this.db, kb.id, {
|
|
886
1147
|
phase: "planning",
|
|
@@ -899,7 +1160,7 @@ export class KnowledgeEngine {
|
|
|
899
1160
|
onProgress?.(scanningMessage);
|
|
900
1161
|
const skipped = createSkippedScanStats();
|
|
901
1162
|
let processedFiles = 0;
|
|
902
|
-
for (const file of
|
|
1163
|
+
for await (const file of iterateScannableFilesAsync(resolvedSource, skipped, scanOptions, signal)) {
|
|
903
1164
|
if (signal?.aborted)
|
|
904
1165
|
throw new Error("Cancelled");
|
|
905
1166
|
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
@@ -932,6 +1193,9 @@ export class KnowledgeEngine {
|
|
|
932
1193
|
}
|
|
933
1194
|
else {
|
|
934
1195
|
fileCount = 1;
|
|
1196
|
+
if (Buffer.byteLength(source) > MAX_DIRECT_TEXT_BYTES) {
|
|
1197
|
+
throw new Error(`Inline text exceeds the ${MAX_DIRECT_TEXT_BYTES}-byte indexing limit`);
|
|
1198
|
+
}
|
|
935
1199
|
const chunks = await analyzeAndAddSymbols(source, "inline-text", "text");
|
|
936
1200
|
await addChunks(chunks);
|
|
937
1201
|
}
|
|
@@ -942,6 +1206,7 @@ export class KnowledgeEngine {
|
|
|
942
1206
|
tempVectorFile = undefined;
|
|
943
1207
|
const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
|
|
944
1208
|
updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
|
|
1209
|
+
markKBIndexed(this.db, kb.id);
|
|
945
1210
|
updateKBStatus(this.db, kb.id, "ready");
|
|
946
1211
|
const savedKB = getKB(this.db, kb.id);
|
|
947
1212
|
if (!savedKB)
|
|
@@ -972,7 +1237,7 @@ export class KnowledgeEngine {
|
|
|
972
1237
|
throw e;
|
|
973
1238
|
}
|
|
974
1239
|
}
|
|
975
|
-
async update(nameOrId, onProgress, signal) {
|
|
1240
|
+
async update(nameOrId, onProgress, signal, access) {
|
|
976
1241
|
if (!this.db)
|
|
977
1242
|
throw new Error("Engine not initialized");
|
|
978
1243
|
if (this.disposing)
|
|
@@ -984,24 +1249,32 @@ export class KnowledgeEngine {
|
|
|
984
1249
|
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
985
1250
|
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
986
1251
|
}
|
|
987
|
-
|
|
1252
|
+
let sourcePath;
|
|
1253
|
+
if (kb.source_type === "url") {
|
|
1254
|
+
assertNetworkAccess(access);
|
|
1255
|
+
sourcePath = kb.source_path;
|
|
1256
|
+
}
|
|
1257
|
+
else {
|
|
1258
|
+
sourcePath = authorizeExistingPath(kb.source_path, access);
|
|
1259
|
+
}
|
|
1260
|
+
const updateableKB = { ...kb, source_path: sourcePath };
|
|
988
1261
|
const activeUpdate = this.activeUpdates.get(updateableKB.id);
|
|
989
1262
|
if (activeUpdate) {
|
|
990
1263
|
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
991
1264
|
return activeUpdate;
|
|
992
1265
|
}
|
|
993
|
-
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
|
|
1266
|
+
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal, access)).finally(() => {
|
|
994
1267
|
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
995
1268
|
this.activeUpdates.delete(updateableKB.id);
|
|
996
1269
|
});
|
|
997
1270
|
this.activeUpdates.set(updateableKB.id, updateRun);
|
|
998
1271
|
return updateRun;
|
|
999
1272
|
}
|
|
1000
|
-
async runUpdate(kb, onProgress, signal) {
|
|
1273
|
+
async runUpdate(kb, onProgress, signal, access) {
|
|
1001
1274
|
if (!this.db)
|
|
1002
1275
|
throw new Error("Engine not initialized");
|
|
1003
1276
|
throwIfAborted(signal);
|
|
1004
|
-
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
1277
|
+
const scanOptions = toScanOptions(parseAddOptions(kb.source_options), access);
|
|
1005
1278
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
1006
1279
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
1007
1280
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
@@ -1140,8 +1413,8 @@ export class KnowledgeEngine {
|
|
|
1140
1413
|
stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
|
|
1141
1414
|
await processChunks(chunks);
|
|
1142
1415
|
}
|
|
1143
|
-
else if (
|
|
1144
|
-
const plan =
|
|
1416
|
+
else if (kb.source_type === "directory") {
|
|
1417
|
+
const plan = await planDirectoryScanAsync(kb.source_path, scanOptions, signal);
|
|
1145
1418
|
plannedTotalFiles = plan.files;
|
|
1146
1419
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
1147
1420
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -1152,12 +1425,27 @@ export class KnowledgeEngine {
|
|
|
1152
1425
|
});
|
|
1153
1426
|
onProgress?.(planningMessage);
|
|
1154
1427
|
const skipped = createSkippedScanStats();
|
|
1155
|
-
for (const file of
|
|
1428
|
+
for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptions, signal)) {
|
|
1156
1429
|
if (signal?.aborted)
|
|
1157
1430
|
throw new Error("Cancelled");
|
|
1158
1431
|
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
1159
|
-
if (!extracted)
|
|
1432
|
+
if (!extracted) {
|
|
1433
|
+
for (const chunk of getChunksByFile(this.db, kb.id, file.relPath, 0, Number.MAX_SAFE_INTEGER)) {
|
|
1434
|
+
const entries = existingHashes.get(chunk.content_hash);
|
|
1435
|
+
const retainedIndex = entries?.findIndex((entry) => entry.id === chunk.id) ?? -1;
|
|
1436
|
+
if (!entries || retainedIndex < 0)
|
|
1437
|
+
continue;
|
|
1438
|
+
const [retained] = entries.splice(retainedIndex, 1);
|
|
1439
|
+
if (canReuseExistingVectors && retained) {
|
|
1440
|
+
const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
|
|
1441
|
+
indexes.push(retained.vectorIndex);
|
|
1442
|
+
oldVectorIndexByHash.set(chunk.content_hash, indexes);
|
|
1443
|
+
}
|
|
1444
|
+
unchanged++;
|
|
1445
|
+
}
|
|
1446
|
+
stagedSymbols.push(...getSymbolsByFile(this.db, kb.id, file.relPath));
|
|
1160
1447
|
continue;
|
|
1448
|
+
}
|
|
1161
1449
|
const analysis = await analyzeIndexableContent(extracted.content, file.relPath, extracted.fileType);
|
|
1162
1450
|
scannedFiles++;
|
|
1163
1451
|
stagedSymbols.push(...analysis.symbols);
|
|
@@ -1193,7 +1481,7 @@ export class KnowledgeEngine {
|
|
|
1193
1481
|
onProgress?.(message);
|
|
1194
1482
|
}
|
|
1195
1483
|
else {
|
|
1196
|
-
const extracted = await extractSourceFileContent(kb.source_path, signal);
|
|
1484
|
+
const extracted = await extractSourceFileContent(kb.source_path, signal, undefined, access);
|
|
1197
1485
|
scannedFiles = 1;
|
|
1198
1486
|
const analysis = await analyzeIndexableContent(extracted.content, kb.source_path, extracted.fileType);
|
|
1199
1487
|
stagedSymbols.push(...analysis.symbols);
|
|
@@ -1300,6 +1588,7 @@ export class KnowledgeEngine {
|
|
|
1300
1588
|
updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
|
|
1301
1589
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
1302
1590
|
updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
|
|
1591
|
+
markKBIndexed(this.db, kb.id);
|
|
1303
1592
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1304
1593
|
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
1305
1594
|
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
|
|
@@ -1616,7 +1905,7 @@ export class KnowledgeEngine {
|
|
|
1616
1905
|
const page = diversified.slice(offset, offset + limit);
|
|
1617
1906
|
const results = page.map((r) => {
|
|
1618
1907
|
const kb = kbById.get(r.chunk.kb_id);
|
|
1619
|
-
const
|
|
1908
|
+
const stale = kb?.status === "stale";
|
|
1620
1909
|
return {
|
|
1621
1910
|
content: r.content,
|
|
1622
1911
|
file_path: r.chunk.file_path,
|
|
@@ -1631,8 +1920,7 @@ export class KnowledgeEngine {
|
|
|
1631
1920
|
chunk_id: r.chunk.id,
|
|
1632
1921
|
chunk_hash: r.chunk.content_hash,
|
|
1633
1922
|
indexed_at: r.chunk.indexed_at,
|
|
1634
|
-
|
|
1635
|
-
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1923
|
+
stale,
|
|
1636
1924
|
match_reason: matchReasonFor(mode),
|
|
1637
1925
|
source_chunk_ids: r.sourceChunkIds,
|
|
1638
1926
|
},
|
|
@@ -1730,13 +2018,12 @@ export class KnowledgeEngine {
|
|
|
1730
2018
|
end_line: r.endLine,
|
|
1731
2019
|
provenance: (() => {
|
|
1732
2020
|
const kb = kbById.get(r.chunk.kb_id);
|
|
1733
|
-
const
|
|
2021
|
+
const stale = kb?.status === "stale";
|
|
1734
2022
|
return {
|
|
1735
2023
|
chunk_id: r.chunk.id,
|
|
1736
2024
|
chunk_hash: r.chunk.content_hash,
|
|
1737
2025
|
indexed_at: r.chunk.indexed_at,
|
|
1738
|
-
|
|
1739
|
-
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
2026
|
+
stale,
|
|
1740
2027
|
match_reason: matchReasonFor(mode),
|
|
1741
2028
|
source_chunk_ids: r.sourceChunkIds,
|
|
1742
2029
|
};
|
|
@@ -1794,11 +2081,11 @@ export class KnowledgeEngine {
|
|
|
1794
2081
|
this.deleteVectorFile(kb.id);
|
|
1795
2082
|
return true;
|
|
1796
2083
|
}
|
|
1797
|
-
list(signal) {
|
|
2084
|
+
list(signal, limit) {
|
|
1798
2085
|
if (!this.db)
|
|
1799
2086
|
return [];
|
|
1800
2087
|
throwIfAborted(signal);
|
|
1801
|
-
return listKBs(this.db);
|
|
2088
|
+
return listKBs(this.db, limit);
|
|
1802
2089
|
}
|
|
1803
2090
|
clear() {
|
|
1804
2091
|
if (!this.db)
|
|
@@ -1822,16 +2109,32 @@ export class KnowledgeEngine {
|
|
|
1822
2109
|
return diagnoseKB(db, kb, signal);
|
|
1823
2110
|
});
|
|
1824
2111
|
}
|
|
1825
|
-
|
|
2112
|
+
async diagnoseAsync(signal, knowledgeBasesOverride, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
1826
2113
|
if (!this.db)
|
|
1827
|
-
return
|
|
2114
|
+
return [];
|
|
1828
2115
|
throwIfAborted(signal);
|
|
1829
2116
|
const db = this.db;
|
|
1830
|
-
const
|
|
1831
|
-
|
|
2117
|
+
const diagnostics = [];
|
|
2118
|
+
let remainingPaths = maxDiagnosticPaths;
|
|
2119
|
+
for (const kb of knowledgeBasesOverride ?? listKBs(db)) {
|
|
1832
2120
|
throwIfAborted(signal);
|
|
1833
|
-
|
|
1834
|
-
|
|
2121
|
+
const diagnostic = await diagnoseKBAsync(db, kb, signal, Math.max(0, remainingPaths));
|
|
2122
|
+
diagnostics.push(diagnostic);
|
|
2123
|
+
remainingPaths = Math.max(0, remainingPaths - diagnostic.stale_files.length - diagnostic.orphan_files.length);
|
|
2124
|
+
}
|
|
2125
|
+
return diagnostics;
|
|
2126
|
+
}
|
|
2127
|
+
statusSnapshot(signal, diagnosticsOverride, knowledgeBasesOverride) {
|
|
2128
|
+
if (!this.db)
|
|
2129
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
2130
|
+
throwIfAborted(signal);
|
|
2131
|
+
const db = this.db;
|
|
2132
|
+
const knowledgeBases = knowledgeBasesOverride ?? listKBs(db);
|
|
2133
|
+
const diagnostics = diagnosticsOverride ??
|
|
2134
|
+
knowledgeBases.map((kb) => {
|
|
2135
|
+
throwIfAborted(signal);
|
|
2136
|
+
return diagnoseKB(db, kb, signal);
|
|
2137
|
+
});
|
|
1835
2138
|
try {
|
|
1836
2139
|
const config = resolveEmbeddingConfig();
|
|
1837
2140
|
const activeModel = embeddingConfigLabel(config);
|
|
@@ -1862,6 +2165,15 @@ export class KnowledgeEngine {
|
|
|
1862
2165
|
};
|
|
1863
2166
|
}
|
|
1864
2167
|
}
|
|
2168
|
+
async statusSnapshotAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
2169
|
+
if (!this.db)
|
|
2170
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
2171
|
+
const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
|
|
2172
|
+
if (knowledgeBases.length > maxKnowledgeBases) {
|
|
2173
|
+
throw new Error(`Knowledge status exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
|
|
2174
|
+
}
|
|
2175
|
+
return this.statusSnapshot(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
|
|
2176
|
+
}
|
|
1865
2177
|
symbolSearch(query, options = {}, signal) {
|
|
1866
2178
|
if (!this.db)
|
|
1867
2179
|
throw new Error("Engine not initialized");
|
|
@@ -1952,11 +2264,11 @@ export class KnowledgeEngine {
|
|
|
1952
2264
|
miss,
|
|
1953
2265
|
};
|
|
1954
2266
|
}
|
|
1955
|
-
doctor(signal) {
|
|
2267
|
+
doctor(signal, diagnosticsOverride, knowledgeBasesOverride) {
|
|
1956
2268
|
throwIfAborted(signal);
|
|
1957
|
-
const diagnostics = this.diagnose(signal);
|
|
2269
|
+
const diagnostics = diagnosticsOverride ?? this.diagnose(signal);
|
|
1958
2270
|
const issues = [];
|
|
1959
|
-
const kbs = this.list();
|
|
2271
|
+
const kbs = knowledgeBasesOverride ?? this.list();
|
|
1960
2272
|
if (kbs.length === 0) {
|
|
1961
2273
|
issues.push({
|
|
1962
2274
|
severity: "blocking",
|
|
@@ -1986,7 +2298,16 @@ export class KnowledgeEngine {
|
|
|
1986
2298
|
action_code: "rebuild_kb",
|
|
1987
2299
|
});
|
|
1988
2300
|
}
|
|
1989
|
-
if (diagnostic.
|
|
2301
|
+
if (diagnostic.scan_error) {
|
|
2302
|
+
issues.push({
|
|
2303
|
+
severity: "warning",
|
|
2304
|
+
kb_name: diagnostic.kb_name,
|
|
2305
|
+
message: `Source diagnostics scan failed: ${diagnostic.scan_error}`,
|
|
2306
|
+
action: "Check source accessibility and scanner limits, then run knowledge_doctor again.",
|
|
2307
|
+
action_code: "check_source",
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
else if (diagnostic.coverage_percent < 80) {
|
|
1990
2311
|
issues.push({
|
|
1991
2312
|
severity: "warning",
|
|
1992
2313
|
kb_name: diagnostic.kb_name,
|
|
@@ -1995,20 +2316,22 @@ export class KnowledgeEngine {
|
|
|
1995
2316
|
action_code: "review_skipped_scope",
|
|
1996
2317
|
});
|
|
1997
2318
|
}
|
|
1998
|
-
|
|
2319
|
+
const staleFileCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
|
|
2320
|
+
if (staleFileCount > 0) {
|
|
1999
2321
|
issues.push({
|
|
2000
2322
|
severity: "warning",
|
|
2001
2323
|
kb_name: diagnostic.kb_name,
|
|
2002
|
-
message: `${
|
|
2324
|
+
message: `${staleFileCount} files changed after indexing.`,
|
|
2003
2325
|
action: "Run knowledge_update for this KB.",
|
|
2004
2326
|
action_code: "run_update",
|
|
2005
2327
|
});
|
|
2006
2328
|
}
|
|
2007
|
-
|
|
2329
|
+
const orphanFileCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
|
|
2330
|
+
if (orphanFileCount > 0) {
|
|
2008
2331
|
issues.push({
|
|
2009
2332
|
severity: "warning",
|
|
2010
2333
|
kb_name: diagnostic.kb_name,
|
|
2011
|
-
message: `${
|
|
2334
|
+
message: `${orphanFileCount} indexed files no longer exist in the source.`,
|
|
2012
2335
|
action: "Run knowledge_update or rebuild the KB.",
|
|
2013
2336
|
action_code: "run_update",
|
|
2014
2337
|
});
|
|
@@ -2117,23 +2440,68 @@ export class KnowledgeEngine {
|
|
|
2117
2440
|
})),
|
|
2118
2441
|
};
|
|
2119
2442
|
}
|
|
2120
|
-
async
|
|
2443
|
+
async doctorAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
2444
|
+
if (!this.db)
|
|
2445
|
+
return this.doctor(signal, [], []);
|
|
2446
|
+
const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
|
|
2447
|
+
if (knowledgeBases.length > maxKnowledgeBases) {
|
|
2448
|
+
throw new Error(`Knowledge doctor exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
|
|
2449
|
+
}
|
|
2450
|
+
return this.doctor(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
|
|
2451
|
+
}
|
|
2452
|
+
async exportKB(nameOrId, outputPath, signal, onProgress, access) {
|
|
2121
2453
|
if (!this.db)
|
|
2122
2454
|
throw new Error("Engine not initialized");
|
|
2123
2455
|
throwIfAborted(signal);
|
|
2124
2456
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
2125
2457
|
if (!kb)
|
|
2126
2458
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
2127
|
-
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
|
|
2459
|
+
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress, access));
|
|
2128
2460
|
}
|
|
2129
|
-
async exportKBUnlocked(kb, outputPath, signal, onProgress) {
|
|
2461
|
+
async exportKBUnlocked(kb, outputPath, signal, onProgress, access) {
|
|
2130
2462
|
if (!this.db)
|
|
2131
2463
|
throw new Error("Engine not initialized");
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2464
|
+
if (kb.chunk_count > MAX_EXPORT_CHUNKS) {
|
|
2465
|
+
throw new Error(`Export exceeds the ${MAX_EXPORT_CHUNKS}-chunk limit`);
|
|
2466
|
+
}
|
|
2467
|
+
const target = prepareSafeNewFile(authorizeOutputPath(outputPath, access));
|
|
2468
|
+
let tempFd;
|
|
2469
|
+
let tempDevice;
|
|
2470
|
+
let tempInode;
|
|
2471
|
+
try {
|
|
2472
|
+
tempFd = openSync(target.tempPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
2473
|
+
const opened = fstatSync(tempFd);
|
|
2474
|
+
if (!opened.isFile())
|
|
2475
|
+
throw new Error(`Export target is not a regular file: ${target.tempPath}`);
|
|
2476
|
+
tempDevice = opened.dev;
|
|
2477
|
+
tempInode = opened.ino;
|
|
2478
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2479
|
+
assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
|
|
2480
|
+
}
|
|
2481
|
+
catch (error) {
|
|
2482
|
+
if (tempFd !== undefined)
|
|
2483
|
+
closeSync(tempFd);
|
|
2484
|
+
throw error;
|
|
2485
|
+
}
|
|
2486
|
+
const stream = createWriteStream("", {
|
|
2487
|
+
autoClose: false,
|
|
2488
|
+
encoding: "utf-8",
|
|
2489
|
+
fd: tempFd,
|
|
2490
|
+
});
|
|
2135
2491
|
const progressBatchSize = resolveIndexEmbedBatchSize();
|
|
2136
2492
|
let count = 0;
|
|
2493
|
+
let bytes = 0;
|
|
2494
|
+
const deadline = Date.now() + MAX_EXPORT_DURATION_MS;
|
|
2495
|
+
const writeBoundedLine = async (line) => {
|
|
2496
|
+
if (Date.now() > deadline)
|
|
2497
|
+
throw new Error(`Export exceeded ${MAX_EXPORT_DURATION_MS}ms`);
|
|
2498
|
+
const lineBytes = Buffer.byteLength(line) + 1;
|
|
2499
|
+
if (bytes + lineBytes > MAX_EXPORT_BYTES) {
|
|
2500
|
+
throw new Error(`Export exceeds the ${MAX_EXPORT_BYTES}-byte limit`);
|
|
2501
|
+
}
|
|
2502
|
+
bytes += lineBytes;
|
|
2503
|
+
await writeLine(stream, line);
|
|
2504
|
+
};
|
|
2137
2505
|
const header = JSON.stringify({
|
|
2138
2506
|
name: kb.name,
|
|
2139
2507
|
description: kb.description,
|
|
@@ -2141,48 +2509,130 @@ export class KnowledgeEngine {
|
|
|
2141
2509
|
chunk_count: kb.chunk_count,
|
|
2142
2510
|
});
|
|
2143
2511
|
try {
|
|
2144
|
-
await
|
|
2512
|
+
await writeBoundedLine(header);
|
|
2145
2513
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
2146
2514
|
throwIfAborted(signal);
|
|
2147
|
-
|
|
2515
|
+
const line = JSON.stringify({
|
|
2148
2516
|
content: chunk.content,
|
|
2149
2517
|
file_path: chunk.file_path,
|
|
2150
2518
|
file_type: chunk.file_type,
|
|
2151
2519
|
start_line: chunk.start_line,
|
|
2152
2520
|
end_line: chunk.end_line,
|
|
2153
2521
|
metadata_json: chunk.metadata_json,
|
|
2154
|
-
})
|
|
2522
|
+
});
|
|
2523
|
+
await writeBoundedLine(line);
|
|
2155
2524
|
count++;
|
|
2156
2525
|
if (count % progressBatchSize === 0)
|
|
2157
2526
|
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
|
|
2158
2527
|
}
|
|
2159
2528
|
await finishWriteStream(stream);
|
|
2160
2529
|
throwIfAborted(signal);
|
|
2161
|
-
|
|
2530
|
+
fsyncSync(tempFd);
|
|
2531
|
+
if (stream.fd === tempFd)
|
|
2532
|
+
stream.fd = null;
|
|
2533
|
+
closeSync(tempFd);
|
|
2534
|
+
tempFd = undefined;
|
|
2535
|
+
stream.destroy();
|
|
2536
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2537
|
+
assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
|
|
2538
|
+
publishNewFile(target.tempPath, target);
|
|
2162
2539
|
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
|
|
2163
2540
|
return count;
|
|
2164
2541
|
}
|
|
2165
2542
|
catch (error) {
|
|
2543
|
+
if (tempFd !== undefined) {
|
|
2544
|
+
const descriptor = tempFd;
|
|
2545
|
+
tempFd = undefined;
|
|
2546
|
+
if (stream.fd === descriptor)
|
|
2547
|
+
stream.fd = null;
|
|
2548
|
+
try {
|
|
2549
|
+
closeSync(descriptor);
|
|
2550
|
+
}
|
|
2551
|
+
catch (closeError) {
|
|
2552
|
+
if (closeError.code !== "EBADF") {
|
|
2553
|
+
console.error(`Failed to close export descriptor: ${String(closeError)}`);
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2166
2557
|
stream.destroy();
|
|
2167
|
-
|
|
2558
|
+
try {
|
|
2559
|
+
const current = lstatSync(target.tempPath);
|
|
2560
|
+
if (!current.isSymbolicLink() && current.dev === tempDevice && current.ino === tempInode) {
|
|
2561
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2562
|
+
rmSync(target.tempPath, { force: true });
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2565
|
+
catch {
|
|
2566
|
+
// Leave an unverified path untouched rather than deleting outside the authorized root.
|
|
2567
|
+
}
|
|
2168
2568
|
throw error;
|
|
2169
2569
|
}
|
|
2170
2570
|
}
|
|
2171
|
-
async importKB(inputPath, onProgress, signal) {
|
|
2571
|
+
async importKB(inputPath, onProgress, signal, access) {
|
|
2172
2572
|
if (!this.db)
|
|
2173
2573
|
throw new Error("Engine not initialized");
|
|
2174
|
-
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
|
|
2574
|
+
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal, access));
|
|
2175
2575
|
}
|
|
2176
|
-
async importKBUnlocked(inputPath, onProgress, signal) {
|
|
2576
|
+
async importKBUnlocked(inputPath, onProgress, signal, access) {
|
|
2177
2577
|
if (!this.db)
|
|
2178
2578
|
throw new Error("Engine not initialized");
|
|
2179
2579
|
throwIfAborted(signal);
|
|
2180
|
-
const { createReadStream } = await import("node:fs");
|
|
2181
|
-
const { createInterface } = await import("node:readline");
|
|
2182
2580
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
2183
2581
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
2184
|
-
|
|
2582
|
+
inputPath = authorizeExistingPath(inputPath, access);
|
|
2583
|
+
const directInput = lstatSync(inputPath);
|
|
2584
|
+
if (directInput.isSymbolicLink() || !directInput.isFile()) {
|
|
2585
|
+
throw new Error(`Import source is not a regular non-symlink file: ${inputPath}`);
|
|
2586
|
+
}
|
|
2587
|
+
const inputFd = openSync(inputPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
2588
|
+
const inputInfo = fstatSync(inputFd);
|
|
2589
|
+
if (inputInfo.dev !== directInput.dev || inputInfo.ino !== directInput.ino) {
|
|
2590
|
+
closeSync(inputFd);
|
|
2591
|
+
throw new Error(`Import source changed while opening: ${inputPath}`);
|
|
2592
|
+
}
|
|
2593
|
+
if (!inputInfo.isFile()) {
|
|
2594
|
+
closeSync(inputFd);
|
|
2595
|
+
throw new Error(`Import source is not a regular file: ${inputPath}`);
|
|
2596
|
+
}
|
|
2597
|
+
try {
|
|
2598
|
+
assertOpenedPathAuthorized(inputPath, inputInfo.dev, inputInfo.ino, access);
|
|
2599
|
+
}
|
|
2600
|
+
catch (error) {
|
|
2601
|
+
closeSync(inputFd);
|
|
2602
|
+
throw error;
|
|
2603
|
+
}
|
|
2604
|
+
if (inputInfo.size > MAX_IMPORT_BYTES) {
|
|
2605
|
+
closeSync(inputFd);
|
|
2606
|
+
throw new Error(`Import file exceeds the ${MAX_IMPORT_BYTES}-byte limit: ${inputPath}`);
|
|
2607
|
+
}
|
|
2608
|
+
const stream = createReadStream("", { autoClose: true, encoding: "utf-8", fd: inputFd });
|
|
2185
2609
|
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
2610
|
+
let observedBytes = 0;
|
|
2611
|
+
let observedLineBytes = 0;
|
|
2612
|
+
stream.on("data", (chunk) => {
|
|
2613
|
+
observedBytes += Buffer.byteLength(String(chunk));
|
|
2614
|
+
if (observedBytes > MAX_IMPORT_BYTES) {
|
|
2615
|
+
stream.destroy(new Error(`Import exceeds the ${MAX_IMPORT_BYTES}-byte limit`));
|
|
2616
|
+
return;
|
|
2617
|
+
}
|
|
2618
|
+
const parts = String(chunk).split("\n");
|
|
2619
|
+
if (parts.length === 1) {
|
|
2620
|
+
observedLineBytes += Buffer.byteLength(parts[0] ?? "");
|
|
2621
|
+
}
|
|
2622
|
+
else {
|
|
2623
|
+
observedLineBytes += Buffer.byteLength(parts[0] ?? "");
|
|
2624
|
+
for (let index = 1; index < parts.length - 1; index++) {
|
|
2625
|
+
if (Buffer.byteLength(parts[index] ?? "") > MAX_IMPORT_LINE_BYTES) {
|
|
2626
|
+
stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
|
|
2627
|
+
return;
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
observedLineBytes = Buffer.byteLength(parts.at(-1) ?? "");
|
|
2631
|
+
}
|
|
2632
|
+
if (observedLineBytes > MAX_IMPORT_LINE_BYTES) {
|
|
2633
|
+
stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
|
|
2634
|
+
}
|
|
2635
|
+
});
|
|
2186
2636
|
let header;
|
|
2187
2637
|
let kb;
|
|
2188
2638
|
let vectorWriter;
|
|
@@ -2228,13 +2678,28 @@ export class KnowledgeEngine {
|
|
|
2228
2678
|
for await (const rawLine of lines) {
|
|
2229
2679
|
throwIfAborted(signal);
|
|
2230
2680
|
lineNumber++;
|
|
2681
|
+
if (lineNumber > MAX_IMPORT_LINES)
|
|
2682
|
+
throw new Error(`Import exceeds the ${MAX_IMPORT_LINES}-line limit`);
|
|
2231
2683
|
const line = rawLine.trim();
|
|
2684
|
+
if (Buffer.byteLength(rawLine) > MAX_IMPORT_LINE_BYTES) {
|
|
2685
|
+
throw new Error(`Import line ${lineNumber} exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`);
|
|
2686
|
+
}
|
|
2232
2687
|
if (!line)
|
|
2233
2688
|
continue;
|
|
2234
2689
|
if (!header) {
|
|
2235
|
-
|
|
2236
|
-
if (
|
|
2237
|
-
throw new Error("Import header
|
|
2690
|
+
const parsedHeader = JSON.parse(line);
|
|
2691
|
+
if (typeof parsedHeader !== "object" || parsedHeader === null || Array.isArray(parsedHeader)) {
|
|
2692
|
+
throw new Error("Import header must be an object");
|
|
2693
|
+
}
|
|
2694
|
+
const headerRecord = parsedHeader;
|
|
2695
|
+
header = {
|
|
2696
|
+
...validateKnowledgeBaseMetadata(headerRecord.name, headerRecord.description),
|
|
2697
|
+
chunk_count: headerRecord.chunk_count === undefined ? undefined : headerRecord.chunk_count,
|
|
2698
|
+
};
|
|
2699
|
+
if (header.chunk_count !== undefined &&
|
|
2700
|
+
(!Number.isInteger(header.chunk_count) || header.chunk_count < 0 || header.chunk_count > MAX_IMPORT_CHUNKS)) {
|
|
2701
|
+
throw new Error(`Import header chunk_count must be between 0 and ${MAX_IMPORT_CHUNKS}`);
|
|
2702
|
+
}
|
|
2238
2703
|
const existingKB = getKBByName(this.db, header.name);
|
|
2239
2704
|
if (existingKB) {
|
|
2240
2705
|
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
@@ -2260,11 +2725,14 @@ export class KnowledgeEngine {
|
|
|
2260
2725
|
vectorWriter = openVectorWriter(tempVectorFile);
|
|
2261
2726
|
continue;
|
|
2262
2727
|
}
|
|
2263
|
-
const imported = JSON.parse(line);
|
|
2728
|
+
const imported = parseImportedChunk(JSON.parse(line));
|
|
2264
2729
|
const chunk = importedChunkToInsert(imported);
|
|
2265
2730
|
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
2266
2731
|
const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
|
|
2267
2732
|
pendingChunks.push(...finalizedChunks);
|
|
2733
|
+
if (inserted + pendingChunks.length > MAX_IMPORT_CHUNKS) {
|
|
2734
|
+
throw new Error(`Import exceeds the ${MAX_IMPORT_CHUNKS}-chunk limit`);
|
|
2735
|
+
}
|
|
2268
2736
|
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
2269
2737
|
while (pendingChunks.length >= embeddingBatchSize)
|
|
2270
2738
|
await flushPending();
|
|
@@ -2279,6 +2747,7 @@ export class KnowledgeEngine {
|
|
|
2279
2747
|
renameSync(tempVectorFile, this.vectorPathFor(kb.id));
|
|
2280
2748
|
tempVectorFile = undefined;
|
|
2281
2749
|
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
2750
|
+
markKBIndexed(this.db, kb.id);
|
|
2282
2751
|
updateKBStatus(this.db, kb.id, "ready");
|
|
2283
2752
|
const savedKB = getKB(this.db, kb.id);
|
|
2284
2753
|
if (!savedKB)
|