pi-knowledge 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +63 -24
- package/dist/index.d.ts +10 -4
- package/dist/index.js +300 -187
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +63 -9
- package/dist/src/engine.js +959 -170
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +10 -0
- package/dist/src/indexer/chunker.js +284 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +184 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +47 -0
- package/docs/configuration.md +42 -10
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
package/dist/src/engine.js
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
import { existsSync,
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { closeSync, constants, createReadStream, createWriteStream, existsSync, fstatSync, fsyncSync, lstatSync, openSync, renameSync, rmSync, statSync, } from "node:fs";
|
|
2
|
+
import { lstat as lstatFile, mkdtemp, open as openFile, realpath as realpathFile, rm as rmAsync, writeFile, } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
|
+
import { createInterface } from "node:readline";
|
|
6
|
+
import { diagnoseKB, diagnoseKBAsync } from "./diagnostics/health.js";
|
|
4
7
|
import { describeEmbeddingCompatibility, describeEmbeddingDecoration, dispose as disposeEmbedding, embedDocuments, embeddingConfigLabel, embeddingSignature, embedQueryWithConfig, embedValidatedDocuments, inspectEmbeddingCompatibility, prepareForShutdown as prepareEmbeddingForShutdown, resolveEmbeddingConfig, } from "./embedding/provider.js";
|
|
5
8
|
import { openVectorReader, openVectorWriter } from "./embedding/vectors.js";
|
|
6
|
-
import {
|
|
9
|
+
import { assertNetworkAccess, assertOpenedPathAuthorized, assertOutputPathAuthorized, authorizeExistingPath, authorizeOutputPath, } from "./filesystem-access.js";
|
|
10
|
+
import { addSkippedScanEntry, analyzeIndexableContent, buildChunkEmbeddingText, chunkFile, chunkIdentityHash, chunkMarkdown, createSkippedScanStats, isReadableTextFile, isSupportedDocumentFile, iterateScannableFiles, iterateScannableFilesAsync, preTokenizeForFTS, summarizeSkippedScan, } from "./indexer/chunker.js";
|
|
7
11
|
import { finalizeChunksForEmbedding } from "./indexer/embedding-safe-chunks.js";
|
|
8
12
|
import { htmlToStructuredMarkdown } from "./indexer/html.js";
|
|
9
13
|
import { extractSymbols } from "./indexer/symbols.js";
|
|
10
14
|
import { extractDocumentInModelWorker, shutdownModelWorker } from "./model-worker-client.js";
|
|
15
|
+
import { fetchPublicText } from "./network/safe-fetch.js";
|
|
11
16
|
import { searchBM25 } from "./search/bm25.js";
|
|
12
17
|
import { MAX_SEARCH_WINDOW } from "./search/contracts.js";
|
|
13
18
|
import { diversifyRankedChunks, interleaveByFile } from "./search/diversity.js";
|
|
@@ -17,7 +22,8 @@ import { hasAnyLexicalEvidence, hasEnoughLexicalEvidence, normalizeFileTypeFilte
|
|
|
17
22
|
import { disposeReranker, prepareRerankerForShutdown, rerank } from "./search/reranker.js";
|
|
18
23
|
import { resolveSearchTuning, resolveSearchWindow, summarizeSearchTuning, } from "./search/tuning.js";
|
|
19
24
|
import { searchVectorFile } from "./search/vector.js";
|
|
20
|
-
import {
|
|
25
|
+
import { ensureOwnerOnlyDirectory, prepareSafeNewFile, publishNewFile } from "./secure-files.js";
|
|
26
|
+
import { countSymbols, createKB, deleteChunksByIds, deleteKB, deleteSymbolsByKB, finishIndexingJob, getChunkById, getChunkCount, getChunksByFile, getFileCount, getKB, getKBByName, getSymbolCount, getSymbolsByFile, insertChunks, insertSymbols, iterateChunkIdsByKB, iterateChunksByKB, listKBs, markKBIndexed, openDatabase, searchSymbols, startIndexingJob, updateIndexingJob, updateKBCounts, updateKBEmbeddingMetadata, updateKBSourceOptions, updateKBStatus, } from "./storage/sqlite.js";
|
|
21
27
|
export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.js";
|
|
22
28
|
export const CURRENT_URL_STRUCTURE_POLICY = 1;
|
|
23
29
|
export const CURRENT_EMBEDDING_INPUT_POLICY = 2;
|
|
@@ -63,11 +69,12 @@ function matchReasonFor(mode) {
|
|
|
63
69
|
return "bm25";
|
|
64
70
|
return "hybrid";
|
|
65
71
|
}
|
|
66
|
-
function toScanOptions(options = {}) {
|
|
72
|
+
function toScanOptions(options = {}, accessPolicy) {
|
|
67
73
|
return {
|
|
68
74
|
includeSuggestedText: options.include_suggested_text === true,
|
|
69
75
|
includePaths: options.include_paths,
|
|
70
76
|
excludePaths: options.exclude_paths,
|
|
77
|
+
accessPolicy,
|
|
71
78
|
};
|
|
72
79
|
}
|
|
73
80
|
function serializeAddOptions(options = {}) {
|
|
@@ -131,16 +138,22 @@ function planDirectoryScan(dirPath, options = {}, signal) {
|
|
|
131
138
|
skipped,
|
|
132
139
|
};
|
|
133
140
|
}
|
|
134
|
-
function
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
return undefined;
|
|
141
|
+
async function planDirectoryScanAsync(dirPath, options = {}, signal) {
|
|
142
|
+
const skipped = createSkippedScanStats();
|
|
143
|
+
let files = 0;
|
|
144
|
+
let bytes = 0;
|
|
145
|
+
for await (const file of iterateScannableFilesAsync(dirPath, skipped, options, signal)) {
|
|
146
|
+
throwIfAborted(signal);
|
|
147
|
+
files++;
|
|
148
|
+
bytes += file.size;
|
|
143
149
|
}
|
|
150
|
+
return {
|
|
151
|
+
files,
|
|
152
|
+
bytes,
|
|
153
|
+
skippedTotal: skipped.total,
|
|
154
|
+
skippedSummary: summarizeSkippedScan(skipped),
|
|
155
|
+
skipped,
|
|
156
|
+
};
|
|
144
157
|
}
|
|
145
158
|
function kbTrustMultiplier(kb) {
|
|
146
159
|
let multiplier = kb.status === "stale" ? 0.92 : 1;
|
|
@@ -150,6 +163,181 @@ function kbTrustMultiplier(kb) {
|
|
|
150
163
|
multiplier *= 0.98;
|
|
151
164
|
return multiplier;
|
|
152
165
|
}
|
|
166
|
+
function summarizeKnowledgeBases(kbs) {
|
|
167
|
+
return kbs.map((kb) => ({
|
|
168
|
+
id: kb.id,
|
|
169
|
+
name: kb.name,
|
|
170
|
+
status: kb.status,
|
|
171
|
+
chunk_count: kb.chunk_count,
|
|
172
|
+
}));
|
|
173
|
+
}
|
|
174
|
+
function searchMissAction(reason, mode, availableKnowledgeBases) {
|
|
175
|
+
switch (reason) {
|
|
176
|
+
case "no_knowledge_bases":
|
|
177
|
+
return "add_knowledge_base";
|
|
178
|
+
case "knowledge_base_not_found":
|
|
179
|
+
return "fix_kb_id";
|
|
180
|
+
case "no_searchable_knowledge_bases":
|
|
181
|
+
return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
|
|
182
|
+
? "wait_for_indexing"
|
|
183
|
+
: "fix_knowledge_base";
|
|
184
|
+
case "no_indexed_chunks":
|
|
185
|
+
return availableKnowledgeBases.some((kb) => kb.status === "indexing")
|
|
186
|
+
? "wait_for_indexing"
|
|
187
|
+
: "fix_knowledge_base";
|
|
188
|
+
case "embedding_incompatible":
|
|
189
|
+
return "fix_embedding";
|
|
190
|
+
case "no_retrieval_candidates":
|
|
191
|
+
return mode === "fast" || mode === "code" || mode === "config" || mode === "errors"
|
|
192
|
+
? "try_semantic"
|
|
193
|
+
: mode === "semantic" || mode === "semantic_hybrid"
|
|
194
|
+
? "rephrase"
|
|
195
|
+
: "try_semantic_hybrid";
|
|
196
|
+
case "threshold_filtered":
|
|
197
|
+
return mode === "semantic_hybrid" ? "rephrase" : "try_semantic_hybrid";
|
|
198
|
+
case "evidence_filtered":
|
|
199
|
+
return "try_semantic_hybrid";
|
|
200
|
+
case "metadata_filtered":
|
|
201
|
+
return "relax_filter";
|
|
202
|
+
case "page_out_of_range":
|
|
203
|
+
return "reset_offset";
|
|
204
|
+
case "reranker_empty":
|
|
205
|
+
case "no_match":
|
|
206
|
+
return "rephrase";
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
function classifySearchMiss(facts) {
|
|
210
|
+
const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
|
|
211
|
+
let reason;
|
|
212
|
+
if (facts.allKnowledgeBases.length === 0) {
|
|
213
|
+
reason = "no_knowledge_bases";
|
|
214
|
+
}
|
|
215
|
+
else if (facts.requestedKbId &&
|
|
216
|
+
!facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
|
|
217
|
+
reason = "knowledge_base_not_found";
|
|
218
|
+
}
|
|
219
|
+
else if (facts.searchedKnowledgeBases.length === 0) {
|
|
220
|
+
reason = "no_searchable_knowledge_bases";
|
|
221
|
+
}
|
|
222
|
+
else if (facts.searchedKnowledgeBases.every((kb) => kb.chunk_count === 0)) {
|
|
223
|
+
reason = "no_indexed_chunks";
|
|
224
|
+
}
|
|
225
|
+
else if (facts.retrievedCandidates === 0 &&
|
|
226
|
+
facts.embeddingIncompatibleCount === facts.searchedKnowledgeBases.filter((kb) => kb.chunk_count > 0).length) {
|
|
227
|
+
reason = "embedding_incompatible";
|
|
228
|
+
}
|
|
229
|
+
else if (facts.retrievedCandidates === 0) {
|
|
230
|
+
reason = "no_retrieval_candidates";
|
|
231
|
+
}
|
|
232
|
+
else if (facts.thresholdCandidates === 0) {
|
|
233
|
+
reason = "threshold_filtered";
|
|
234
|
+
}
|
|
235
|
+
else if (facts.eligibleCandidates === 0) {
|
|
236
|
+
reason = "evidence_filtered";
|
|
237
|
+
}
|
|
238
|
+
else if (facts.filteredCandidates === 0 && (facts.fileType || facts.pathPattern)) {
|
|
239
|
+
reason = "metadata_filtered";
|
|
240
|
+
}
|
|
241
|
+
else if (facts.mode === "deep" && facts.rankedCandidates === 0) {
|
|
242
|
+
reason = "reranker_empty";
|
|
243
|
+
}
|
|
244
|
+
else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
|
|
245
|
+
reason = "page_out_of_range";
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
reason = "no_match";
|
|
249
|
+
}
|
|
250
|
+
return {
|
|
251
|
+
reason,
|
|
252
|
+
action: searchMissAction(reason, facts.mode, facts.availableKnowledgeBases),
|
|
253
|
+
requested_kb_id: facts.requestedKbId,
|
|
254
|
+
min_hybrid_score: facts.minimumScore,
|
|
255
|
+
file_type: facts.fileType,
|
|
256
|
+
path_pattern: facts.pathPattern,
|
|
257
|
+
searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
|
|
258
|
+
skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
|
|
259
|
+
retrieved_candidates: facts.retrievedCandidates,
|
|
260
|
+
threshold_candidates: facts.thresholdCandidates,
|
|
261
|
+
eligible_candidates: facts.eligibleCandidates,
|
|
262
|
+
filtered_candidates: facts.filteredCandidates,
|
|
263
|
+
ranked_candidates: facts.rankedCandidates,
|
|
264
|
+
total_results: facts.totalResults,
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function searchAttemptSummary(mode, response) {
|
|
268
|
+
return {
|
|
269
|
+
mode,
|
|
270
|
+
reason: response.miss?.reason,
|
|
271
|
+
retrieved_candidates: response.tuning?.retrieved_candidates ?? 0,
|
|
272
|
+
threshold_candidates: response.tuning?.threshold_candidates ?? 0,
|
|
273
|
+
eligible_candidates: response.tuning?.eligible_candidates ?? 0,
|
|
274
|
+
filtered_candidates: response.tuning?.filtered_candidates ?? 0,
|
|
275
|
+
result_count: response.results.length,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
function symbolMissAction(reason, availableKnowledgeBases) {
|
|
279
|
+
switch (reason) {
|
|
280
|
+
case "no_knowledge_bases":
|
|
281
|
+
return "add_knowledge_base";
|
|
282
|
+
case "knowledge_base_not_found":
|
|
283
|
+
return "fix_kb_id";
|
|
284
|
+
case "no_searchable_knowledge_bases":
|
|
285
|
+
return availableKnowledgeBases.length > 0 && availableKnowledgeBases.every((kb) => kb.status === "indexing")
|
|
286
|
+
? "wait_for_indexing"
|
|
287
|
+
: "fix_knowledge_base";
|
|
288
|
+
case "symbol_index_empty":
|
|
289
|
+
return "rebuild_symbol_index";
|
|
290
|
+
case "exact_name_not_found":
|
|
291
|
+
return "broaden_symbol_search";
|
|
292
|
+
case "metadata_filtered":
|
|
293
|
+
return "relax_filter";
|
|
294
|
+
case "page_out_of_range":
|
|
295
|
+
return "reset_offset";
|
|
296
|
+
case "no_match":
|
|
297
|
+
return "use_text_search";
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
function classifySymbolSearchMiss(facts) {
|
|
301
|
+
const skippedKnowledgeBases = facts.availableKnowledgeBases.filter((kb) => kb.status !== "ready" && kb.status !== "stale");
|
|
302
|
+
let reason;
|
|
303
|
+
if (facts.allKnowledgeBases.length === 0) {
|
|
304
|
+
reason = "no_knowledge_bases";
|
|
305
|
+
}
|
|
306
|
+
else if (facts.requestedKbId &&
|
|
307
|
+
!facts.availableKnowledgeBases.some((kb) => kb.id === facts.requestedKbId || kb.name === facts.requestedKbId)) {
|
|
308
|
+
reason = "knowledge_base_not_found";
|
|
309
|
+
}
|
|
310
|
+
else if (facts.searchedKnowledgeBases.length === 0) {
|
|
311
|
+
reason = "no_searchable_knowledge_bases";
|
|
312
|
+
}
|
|
313
|
+
else if (facts.indexedSymbols === 0) {
|
|
314
|
+
reason = "symbol_index_empty";
|
|
315
|
+
}
|
|
316
|
+
else if (facts.totalResults > 0 && facts.offset >= facts.totalResults) {
|
|
317
|
+
reason = "page_out_of_range";
|
|
318
|
+
}
|
|
319
|
+
else if ((facts.kind || facts.filePattern) && facts.unfilteredMatches > 0) {
|
|
320
|
+
reason = "metadata_filtered";
|
|
321
|
+
}
|
|
322
|
+
else if (facts.exact) {
|
|
323
|
+
reason = "exact_name_not_found";
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
reason = "no_match";
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
reason,
|
|
330
|
+
action: symbolMissAction(reason, facts.availableKnowledgeBases),
|
|
331
|
+
requested_kb_id: facts.requestedKbId,
|
|
332
|
+
kind: facts.kind,
|
|
333
|
+
file_pattern: facts.filePattern,
|
|
334
|
+
exact: facts.exact,
|
|
335
|
+
searched_kbs: summarizeKnowledgeBases(facts.searchedKnowledgeBases),
|
|
336
|
+
skipped_kbs: summarizeKnowledgeBases(skippedKnowledgeBases),
|
|
337
|
+
indexed_symbols: facts.indexedSymbols,
|
|
338
|
+
unfiltered_matches: facts.unfilteredMatches,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
153
341
|
function formatBytes(bytes) {
|
|
154
342
|
if (bytes < 1024)
|
|
155
343
|
return `${bytes} B`;
|
|
@@ -360,35 +548,146 @@ function isWeakAutoResponse(query, response, primaryMode, attemptMode) {
|
|
|
360
548
|
}
|
|
361
549
|
return false;
|
|
362
550
|
}
|
|
551
|
+
const MAX_DIRECT_TEXT_BYTES = 10 * 1024 * 1024;
|
|
552
|
+
const MAX_DIRECT_DOCUMENT_BYTES = 64 * 1024 * 1024;
|
|
553
|
+
const MAX_IMPORT_BYTES = 256 * 1024 * 1024;
|
|
554
|
+
const MAX_IMPORT_LINE_BYTES = 16 * 1024 * 1024;
|
|
555
|
+
const MAX_IMPORT_LINES = 1_000_001;
|
|
556
|
+
const MAX_IMPORT_CHUNKS = 1_000_000;
|
|
557
|
+
const MAX_EXPORT_BYTES = MAX_IMPORT_BYTES;
|
|
558
|
+
const MAX_EXPORT_CHUNKS = MAX_IMPORT_CHUNKS;
|
|
559
|
+
const MAX_EXPORT_DURATION_MS = 60 * 60_000;
|
|
560
|
+
const MAX_KB_NAME_BYTES = 512;
|
|
561
|
+
const MAX_KB_DESCRIPTION_BYTES = 4 * 1024;
|
|
562
|
+
const MAX_IMPORT_CONTENT_BYTES = 12 * 1024 * 1024;
|
|
563
|
+
const MAX_IMPORT_FILE_PATH_BYTES = 4_096;
|
|
564
|
+
const MAX_IMPORT_FILE_TYPE_BYTES = 256;
|
|
565
|
+
const MAX_IMPORT_METADATA_BYTES = 64 * 1024;
|
|
566
|
+
const MAX_IMPORT_METADATA_CONTEXT_BYTES = 8 * 1024;
|
|
567
|
+
const MAX_IMPORT_METADATA_SCOPE_ITEMS = 128;
|
|
568
|
+
const MAX_IMPORT_LINE_NUMBER = 2_147_483_647;
|
|
569
|
+
const MAX_IMPORT_FIXED_CONTEXT_BYTES = 64 * 1024;
|
|
570
|
+
const IMPORTED_CHUNK_KEYS = new Set(["content", "file_path", "file_type", "start_line", "end_line", "metadata_json"]);
|
|
571
|
+
const IMPORTED_CONTEXT_STRING_KEYS = [
|
|
572
|
+
"heading",
|
|
573
|
+
"breadcrumb",
|
|
574
|
+
"language",
|
|
575
|
+
"symbol",
|
|
576
|
+
"function_name",
|
|
577
|
+
"symbol_kind",
|
|
578
|
+
"parent_symbol",
|
|
579
|
+
"signature",
|
|
580
|
+
];
|
|
363
581
|
async function chunkUrl(source, signal) {
|
|
364
|
-
const
|
|
365
|
-
if (!res.ok)
|
|
366
|
-
throw new Error(`Fetch failed: ${res.status}`);
|
|
367
|
-
const html = await res.text();
|
|
582
|
+
const html = await fetchPublicText(source, signal);
|
|
368
583
|
throwIfAborted(signal);
|
|
369
584
|
const text = htmlToStructuredMarkdown(html);
|
|
370
585
|
const structuredChunks = chunkMarkdown(text, source, "html");
|
|
371
586
|
return structuredChunks.length > 0 ? structuredChunks : chunkFile(text, source);
|
|
372
587
|
}
|
|
373
|
-
|
|
588
|
+
function validateKnowledgeBaseMetadata(name, description) {
|
|
589
|
+
if (typeof name !== "string" || !name.trim())
|
|
590
|
+
throw new Error("Knowledge base name must be a non-empty string");
|
|
591
|
+
if (Buffer.byteLength(name) > MAX_KB_NAME_BYTES) {
|
|
592
|
+
throw new Error(`Knowledge base name exceeds the ${MAX_KB_NAME_BYTES}-byte limit`);
|
|
593
|
+
}
|
|
594
|
+
const normalizedDescription = description === null ? undefined : description;
|
|
595
|
+
if (normalizedDescription !== undefined && typeof normalizedDescription !== "string") {
|
|
596
|
+
throw new Error("Knowledge base description must be a string");
|
|
597
|
+
}
|
|
598
|
+
if (typeof normalizedDescription === "string" &&
|
|
599
|
+
Buffer.byteLength(normalizedDescription) > MAX_KB_DESCRIPTION_BYTES) {
|
|
600
|
+
throw new Error(`Knowledge base description exceeds the ${MAX_KB_DESCRIPTION_BYTES}-byte limit`);
|
|
601
|
+
}
|
|
602
|
+
return { name, description: normalizedDescription };
|
|
603
|
+
}
|
|
604
|
+
async function readBoundedRegularFile(filePath, maxBytes, signal, expected, access) {
|
|
605
|
+
if (expected?.rootPath) {
|
|
606
|
+
const canonical = await realpathFile(filePath);
|
|
607
|
+
const rel = relative(expected.rootPath, canonical);
|
|
608
|
+
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
|
|
609
|
+
throw new Error(`Source file escaped its scanned root: ${filePath}`);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
const direct = lstatSync(filePath);
|
|
613
|
+
if (direct.isSymbolicLink() || !direct.isFile())
|
|
614
|
+
throw new Error(`Source is not a regular non-symlink file: ${filePath}`);
|
|
615
|
+
if (expected?.device !== undefined &&
|
|
616
|
+
expected.inode !== undefined &&
|
|
617
|
+
(direct.dev !== expected.device || direct.ino !== expected.inode)) {
|
|
618
|
+
throw new Error(`Source file changed after scanning: ${filePath}`);
|
|
619
|
+
}
|
|
620
|
+
const handle = await openFile(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
621
|
+
try {
|
|
622
|
+
const opened = await handle.stat();
|
|
623
|
+
if (!opened.isFile())
|
|
624
|
+
throw new Error(`Source is not a regular file: ${filePath}`);
|
|
625
|
+
assertOpenedPathAuthorized(filePath, opened.dev, opened.ino, access);
|
|
626
|
+
if (expected?.rootPath) {
|
|
627
|
+
const canonical = await realpathFile(filePath);
|
|
628
|
+
const current = await lstatFile(filePath);
|
|
629
|
+
const rel = relative(expected.rootPath, canonical);
|
|
630
|
+
if (rel === ".." ||
|
|
631
|
+
rel.startsWith(`..${sep}`) ||
|
|
632
|
+
isAbsolute(rel) ||
|
|
633
|
+
current.isSymbolicLink() ||
|
|
634
|
+
current.dev !== opened.dev ||
|
|
635
|
+
current.ino !== opened.ino) {
|
|
636
|
+
throw new Error(`Source file changed outside its scanned root: ${filePath}`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (expected?.device !== undefined &&
|
|
640
|
+
expected.inode !== undefined &&
|
|
641
|
+
(opened.dev !== expected.device || opened.ino !== expected.inode)) {
|
|
642
|
+
throw new Error(`Source file changed while opening: ${filePath}`);
|
|
643
|
+
}
|
|
644
|
+
if (opened.size > maxBytes) {
|
|
645
|
+
throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
|
|
646
|
+
}
|
|
647
|
+
const chunks = [];
|
|
648
|
+
let bytes = 0;
|
|
649
|
+
while (bytes <= maxBytes) {
|
|
650
|
+
throwIfAborted(signal);
|
|
651
|
+
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes - bytes + 1));
|
|
652
|
+
const result = await handle.read(buffer, 0, buffer.length, null);
|
|
653
|
+
if (result.bytesRead === 0)
|
|
654
|
+
break;
|
|
655
|
+
bytes += result.bytesRead;
|
|
656
|
+
if (bytes > maxBytes) {
|
|
657
|
+
throw new Error(`Source file exceeds the ${maxBytes}-byte direct indexing limit: ${filePath}`);
|
|
658
|
+
}
|
|
659
|
+
chunks.push(buffer.subarray(0, result.bytesRead));
|
|
660
|
+
}
|
|
661
|
+
return Buffer.concat(chunks, bytes);
|
|
662
|
+
}
|
|
663
|
+
finally {
|
|
664
|
+
await handle.close();
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
async function extractSourceFileContent(filePath, signal, expected, access) {
|
|
668
|
+
filePath = authorizeExistingPath(filePath, access);
|
|
374
669
|
const lowerPath = filePath.toLowerCase();
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
670
|
+
const isDocument = lowerPath.endsWith(".pdf") || lowerPath.endsWith(".docx") || lowerPath.endsWith(".doc");
|
|
671
|
+
const maxBytes = isDocument ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
|
|
672
|
+
const content = await readBoundedRegularFile(filePath, maxBytes, signal, expected, access);
|
|
673
|
+
if (!isDocument) {
|
|
674
|
+
if (content.subarray(0, 8_000).includes(0x00)) {
|
|
675
|
+
throw new Error(`File is not readable text and has no supported extractor: ${filePath}`);
|
|
676
|
+
}
|
|
677
|
+
return { content: content.toString("utf8"), fileType: "text" };
|
|
380
678
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
679
|
+
const tempDirectory = ensureOwnerOnlyDirectory(await mkdtemp(join(tmpdir(), "pi-knowledge-document-")));
|
|
680
|
+
const tempPath = join(tempDirectory, "source");
|
|
681
|
+
try {
|
|
682
|
+
await writeFile(tempPath, content, { flag: "wx", mode: 0o600 });
|
|
683
|
+
const fileType = lowerPath.endsWith(".pdf") ? "pdf" : "docx";
|
|
684
|
+
const extracted = await extractDocumentInModelWorker(tempPath, fileType, signal);
|
|
384
685
|
throwIfAborted(signal);
|
|
385
686
|
return extracted;
|
|
386
687
|
}
|
|
387
|
-
|
|
388
|
-
|
|
688
|
+
finally {
|
|
689
|
+
await rmAsync(tempDirectory, { recursive: true, force: true });
|
|
389
690
|
}
|
|
390
|
-
throwIfAborted(signal);
|
|
391
|
-
return { content: readFileSync(filePath, "utf-8"), fileType: "text" };
|
|
392
691
|
}
|
|
393
692
|
function looksLikeLocalPath(source) {
|
|
394
693
|
const trimmed = source.trim();
|
|
@@ -402,11 +701,15 @@ function looksLikeLocalPath(source) {
|
|
|
402
701
|
return true;
|
|
403
702
|
return /\.[A-Za-z0-9][A-Za-z0-9_-]{0,15}$/.test(trimmed);
|
|
404
703
|
}
|
|
405
|
-
function classifySource(source) {
|
|
406
|
-
const resolvedSource = resolve(source);
|
|
704
|
+
function classifySource(source, access) {
|
|
407
705
|
const isUrl = source.startsWith("http://") || source.startsWith("https://");
|
|
408
|
-
|
|
409
|
-
|
|
706
|
+
if (isUrl)
|
|
707
|
+
assertNetworkAccess(access);
|
|
708
|
+
const requestedSource = resolve(source);
|
|
709
|
+
const exists = !isUrl && existsSync(requestedSource);
|
|
710
|
+
const resolvedSource = exists ? authorizeExistingPath(requestedSource, access) : requestedSource;
|
|
711
|
+
const isDir = exists && statSync(resolvedSource).isDirectory();
|
|
712
|
+
const isFile = exists && statSync(resolvedSource).isFile();
|
|
410
713
|
if (!isUrl && !isDir && !isFile && looksLikeLocalPath(source)) {
|
|
411
714
|
throw new Error(`Local path does not exist: ${source}. Pass inline text only when the source itself is text.`);
|
|
412
715
|
}
|
|
@@ -419,7 +722,7 @@ function classifySource(source) {
|
|
|
419
722
|
};
|
|
420
723
|
}
|
|
421
724
|
async function extractScannableFileContent(file, signal) {
|
|
422
|
-
return extractSourceFileContent(file.path, signal);
|
|
725
|
+
return extractSourceFileContent(file.path, signal, file);
|
|
423
726
|
}
|
|
424
727
|
async function extractScannableFileContentOrSkip(file, skipped, signal) {
|
|
425
728
|
try {
|
|
@@ -447,6 +750,100 @@ function throwIfAborted(signal) {
|
|
|
447
750
|
if (signal?.aborted)
|
|
448
751
|
throw new Error("Cancelled");
|
|
449
752
|
}
|
|
753
|
+
function boundedImportString(value, field, maxBytes) {
|
|
754
|
+
if (typeof value !== "string")
|
|
755
|
+
throw new Error(`Imported chunk ${field} must be a string`);
|
|
756
|
+
if (Buffer.byteLength(value) > maxBytes) {
|
|
757
|
+
throw new Error(`Imported chunk ${field} exceeds the ${maxBytes}-byte limit`);
|
|
758
|
+
}
|
|
759
|
+
return value;
|
|
760
|
+
}
|
|
761
|
+
function validateImportedMetadata(metadataJson) {
|
|
762
|
+
let parsed;
|
|
763
|
+
try {
|
|
764
|
+
parsed = JSON.parse(metadataJson);
|
|
765
|
+
}
|
|
766
|
+
catch {
|
|
767
|
+
throw new Error("Imported chunk metadata_json must contain valid JSON");
|
|
768
|
+
}
|
|
769
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
770
|
+
throw new Error("Imported chunk metadata_json must contain an object");
|
|
771
|
+
}
|
|
772
|
+
const metadata = parsed;
|
|
773
|
+
let contextBytes = 0;
|
|
774
|
+
for (const key of IMPORTED_CONTEXT_STRING_KEYS) {
|
|
775
|
+
const value = metadata[key];
|
|
776
|
+
if (value === undefined)
|
|
777
|
+
continue;
|
|
778
|
+
if (typeof value !== "string")
|
|
779
|
+
throw new Error(`Imported chunk metadata ${key} must be a string`);
|
|
780
|
+
contextBytes += Buffer.byteLength(value);
|
|
781
|
+
}
|
|
782
|
+
const scope = metadata.scope;
|
|
783
|
+
if (scope !== undefined) {
|
|
784
|
+
if (!Array.isArray(scope) || scope.length > MAX_IMPORT_METADATA_SCOPE_ITEMS) {
|
|
785
|
+
throw new Error("Imported chunk metadata scope is invalid");
|
|
786
|
+
}
|
|
787
|
+
for (const item of scope) {
|
|
788
|
+
if (typeof item !== "string")
|
|
789
|
+
throw new Error("Imported chunk metadata scope must contain strings");
|
|
790
|
+
contextBytes += Buffer.byteLength(item);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
if (contextBytes > MAX_IMPORT_METADATA_CONTEXT_BYTES) {
|
|
794
|
+
throw new Error(`Imported chunk embedding metadata exceeds the ${MAX_IMPORT_METADATA_CONTEXT_BYTES}-byte limit`);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
function parseImportedChunk(value) {
|
|
798
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
799
|
+
throw new Error("Imported chunk must be an object");
|
|
800
|
+
}
|
|
801
|
+
const record = value;
|
|
802
|
+
for (const key of Object.keys(record)) {
|
|
803
|
+
if (!IMPORTED_CHUNK_KEYS.has(key))
|
|
804
|
+
throw new Error(`Imported chunk contains unknown field: ${key}`);
|
|
805
|
+
}
|
|
806
|
+
const content = boundedImportString(record.content, "content", MAX_IMPORT_CONTENT_BYTES);
|
|
807
|
+
const filePath = boundedImportString(record.file_path, "file_path", MAX_IMPORT_FILE_PATH_BYTES);
|
|
808
|
+
const fileType = boundedImportString(record.file_type, "file_type", MAX_IMPORT_FILE_TYPE_BYTES);
|
|
809
|
+
if (!content)
|
|
810
|
+
throw new Error("Imported chunk content must not be empty");
|
|
811
|
+
if (!filePath)
|
|
812
|
+
throw new Error("Imported chunk file_path must not be empty");
|
|
813
|
+
if (!fileType)
|
|
814
|
+
throw new Error("Imported chunk file_type must not be empty");
|
|
815
|
+
const startLine = record.start_line;
|
|
816
|
+
const endLine = record.end_line;
|
|
817
|
+
if (!Number.isSafeInteger(startLine) ||
|
|
818
|
+
typeof startLine !== "number" ||
|
|
819
|
+
startLine < 0 ||
|
|
820
|
+
startLine > MAX_IMPORT_LINE_NUMBER) {
|
|
821
|
+
throw new Error("Imported chunk start_line is invalid");
|
|
822
|
+
}
|
|
823
|
+
if (!Number.isSafeInteger(endLine) ||
|
|
824
|
+
typeof endLine !== "number" ||
|
|
825
|
+
endLine < startLine ||
|
|
826
|
+
endLine > MAX_IMPORT_LINE_NUMBER) {
|
|
827
|
+
throw new Error("Imported chunk end_line is invalid");
|
|
828
|
+
}
|
|
829
|
+
const metadataJson = record.metadata_json === undefined
|
|
830
|
+
? "{}"
|
|
831
|
+
: boundedImportString(record.metadata_json, "metadata_json", MAX_IMPORT_METADATA_BYTES);
|
|
832
|
+
validateImportedMetadata(metadataJson);
|
|
833
|
+
const imported = {
|
|
834
|
+
content,
|
|
835
|
+
file_path: filePath,
|
|
836
|
+
file_type: fileType,
|
|
837
|
+
start_line: startLine,
|
|
838
|
+
end_line: endLine,
|
|
839
|
+
metadata_json: metadataJson,
|
|
840
|
+
};
|
|
841
|
+
const fixedContext = buildChunkEmbeddingText({ ...imported, content: "" });
|
|
842
|
+
if (Buffer.byteLength(fixedContext) > MAX_IMPORT_FIXED_CONTEXT_BYTES) {
|
|
843
|
+
throw new Error(`Imported chunk fixed embedding context exceeds the ${MAX_IMPORT_FIXED_CONTEXT_BYTES}-byte limit`);
|
|
844
|
+
}
|
|
845
|
+
return imported;
|
|
846
|
+
}
|
|
450
847
|
function importedChunkToInsert(chunk) {
|
|
451
848
|
const metadataJson = chunk.metadata_json || "{}";
|
|
452
849
|
return {
|
|
@@ -543,9 +940,12 @@ export class KnowledgeEngine {
|
|
|
543
940
|
deleteVectorFile(kbId) {
|
|
544
941
|
rmSync(this.vectorPathFor(kbId), { force: true });
|
|
545
942
|
}
|
|
546
|
-
plan(source, options = {}, signal) {
|
|
943
|
+
plan(source, options = {}, signal, access) {
|
|
547
944
|
throwIfAborted(signal);
|
|
548
|
-
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source);
|
|
945
|
+
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
|
|
946
|
+
if ((isDir || isFile) && lstatSync(resolvedSource).isSymbolicLink()) {
|
|
947
|
+
throw new Error(`Source is not a regular non-symlink path: ${resolvedSource}`);
|
|
948
|
+
}
|
|
549
949
|
if (isDir) {
|
|
550
950
|
const plan = planDirectoryScan(resolvedSource, toScanOptions(options), signal);
|
|
551
951
|
return {
|
|
@@ -589,17 +989,53 @@ export class KnowledgeEngine {
|
|
|
589
989
|
summary: `${sourceType === "url" ? "URL" : "Inline text"} plan: 1 source`,
|
|
590
990
|
};
|
|
591
991
|
}
|
|
592
|
-
async
|
|
992
|
+
async planAsync(source, options = {}, signal, access) {
|
|
593
993
|
throwIfAborted(signal);
|
|
594
|
-
|
|
994
|
+
const { resolvedSource, isDir, isFile, sourceType } = classifySource(source, access);
|
|
995
|
+
if (isDir) {
|
|
996
|
+
const plan = await planDirectoryScanAsync(resolvedSource, toScanOptions(options, access), signal);
|
|
997
|
+
return {
|
|
998
|
+
source_type: "directory",
|
|
999
|
+
scannable_files: plan.files,
|
|
1000
|
+
scannable_bytes: plan.bytes,
|
|
1001
|
+
skipped: plan.skipped,
|
|
1002
|
+
summary: `Directory plan: ${plan.files} scannable files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`,
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
if (isFile && lstatSync(resolvedSource).isSymbolicLink()) {
|
|
1006
|
+
throw new Error(`Source is not a regular non-symlink file: ${resolvedSource}`);
|
|
1007
|
+
}
|
|
1008
|
+
const skipped = createSkippedScanStats();
|
|
1009
|
+
const sourceBytes = isFile ? statSync(resolvedSource).size : Buffer.byteLength(source);
|
|
1010
|
+
const maxBytes = isFile && isSupportedDocumentFile(resolvedSource) ? MAX_DIRECT_DOCUMENT_BYTES : MAX_DIRECT_TEXT_BYTES;
|
|
1011
|
+
if (sourceType !== "url" && sourceBytes > maxBytes) {
|
|
1012
|
+
addSkippedScanEntry(skipped, {
|
|
1013
|
+
path: isFile ? resolvedSource : "<inline>",
|
|
1014
|
+
reason: "oversized",
|
|
1015
|
+
size: sourceBytes,
|
|
1016
|
+
});
|
|
1017
|
+
return {
|
|
1018
|
+
source_type: sourceType,
|
|
1019
|
+
scannable_files: 0,
|
|
1020
|
+
scannable_bytes: 0,
|
|
1021
|
+
skipped,
|
|
1022
|
+
summary: `${isFile ? "File" : "Inline text"} plan: 0 scannable sources; source exceeds ${formatBytes(maxBytes)}`,
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
return this.plan(source, options, signal, access);
|
|
1026
|
+
}
|
|
1027
|
+
async add(source, name, onProgress, signal, options = {}, access) {
|
|
1028
|
+
throwIfAborted(signal);
|
|
1029
|
+
validateKnowledgeBaseMetadata(name, undefined);
|
|
1030
|
+
return this.runExclusive(["global:mutation", `kb-name:${name}`], `Mutation for "${name}"`, () => this.addUnlocked(source, name, onProgress, signal, options, access));
|
|
595
1031
|
}
|
|
596
|
-
async addUnlocked(source, name, onProgress, signal, options = {}) {
|
|
1032
|
+
async addUnlocked(source, name, onProgress, signal, options = {}, access) {
|
|
597
1033
|
if (!this.db)
|
|
598
1034
|
throw new Error("Engine not initialized");
|
|
599
1035
|
throwIfAborted(signal);
|
|
600
1036
|
const db = this.db;
|
|
601
|
-
const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source);
|
|
602
|
-
const scanOptions = toScanOptions(options);
|
|
1037
|
+
const { resolvedSource, isUrl, isDir, isFile, sourceType } = classifySource(source, access);
|
|
1038
|
+
const scanOptions = toScanOptions(options, access);
|
|
603
1039
|
const existingKB = getKBByName(db, name);
|
|
604
1040
|
if (existingKB) {
|
|
605
1041
|
throw new Error(`Knowledge base "${name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before adding a replacement.`);
|
|
@@ -699,13 +1135,13 @@ export class KnowledgeEngine {
|
|
|
699
1135
|
await addChunks(chunks);
|
|
700
1136
|
}
|
|
701
1137
|
else if (isFile) {
|
|
702
|
-
const extracted = await extractSourceFileContent(resolvedSource, signal);
|
|
1138
|
+
const extracted = await extractSourceFileContent(resolvedSource, signal, undefined, access);
|
|
703
1139
|
fileCount = 1;
|
|
704
1140
|
const chunks = await analyzeAndAddSymbols(extracted.content, resolvedSource, extracted.fileType);
|
|
705
1141
|
await addChunks(chunks);
|
|
706
1142
|
}
|
|
707
1143
|
else if (isDir) {
|
|
708
|
-
const plan =
|
|
1144
|
+
const plan = await planDirectoryScanAsync(resolvedSource, scanOptions, signal);
|
|
709
1145
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
710
1146
|
updateIndexingJob(this.db, kb.id, {
|
|
711
1147
|
phase: "planning",
|
|
@@ -724,7 +1160,7 @@ export class KnowledgeEngine {
|
|
|
724
1160
|
onProgress?.(scanningMessage);
|
|
725
1161
|
const skipped = createSkippedScanStats();
|
|
726
1162
|
let processedFiles = 0;
|
|
727
|
-
for (const file of
|
|
1163
|
+
for await (const file of iterateScannableFilesAsync(resolvedSource, skipped, scanOptions, signal)) {
|
|
728
1164
|
if (signal?.aborted)
|
|
729
1165
|
throw new Error("Cancelled");
|
|
730
1166
|
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
@@ -757,6 +1193,9 @@ export class KnowledgeEngine {
|
|
|
757
1193
|
}
|
|
758
1194
|
else {
|
|
759
1195
|
fileCount = 1;
|
|
1196
|
+
if (Buffer.byteLength(source) > MAX_DIRECT_TEXT_BYTES) {
|
|
1197
|
+
throw new Error(`Inline text exceeds the ${MAX_DIRECT_TEXT_BYTES}-byte indexing limit`);
|
|
1198
|
+
}
|
|
760
1199
|
const chunks = await analyzeAndAddSymbols(source, "inline-text", "text");
|
|
761
1200
|
await addChunks(chunks);
|
|
762
1201
|
}
|
|
@@ -767,6 +1206,7 @@ export class KnowledgeEngine {
|
|
|
767
1206
|
tempVectorFile = undefined;
|
|
768
1207
|
const savedFileCount = isDir ? getFileCount(this.db, kb.id) : fileCount;
|
|
769
1208
|
updateKBCounts(this.db, kb.id, chunkCount, savedFileCount);
|
|
1209
|
+
markKBIndexed(this.db, kb.id);
|
|
770
1210
|
updateKBStatus(this.db, kb.id, "ready");
|
|
771
1211
|
const savedKB = getKB(this.db, kb.id);
|
|
772
1212
|
if (!savedKB)
|
|
@@ -797,7 +1237,7 @@ export class KnowledgeEngine {
|
|
|
797
1237
|
throw e;
|
|
798
1238
|
}
|
|
799
1239
|
}
|
|
800
|
-
async update(nameOrId, onProgress, signal) {
|
|
1240
|
+
async update(nameOrId, onProgress, signal, access) {
|
|
801
1241
|
if (!this.db)
|
|
802
1242
|
throw new Error("Engine not initialized");
|
|
803
1243
|
if (this.disposing)
|
|
@@ -809,24 +1249,32 @@ export class KnowledgeEngine {
|
|
|
809
1249
|
if (!kb.source_path || (kb.source_type !== "url" && !existsSync(kb.source_path))) {
|
|
810
1250
|
throw new Error(`Source path not available or missing: ${kb.source_path}`);
|
|
811
1251
|
}
|
|
812
|
-
|
|
1252
|
+
let sourcePath;
|
|
1253
|
+
if (kb.source_type === "url") {
|
|
1254
|
+
assertNetworkAccess(access);
|
|
1255
|
+
sourcePath = kb.source_path;
|
|
1256
|
+
}
|
|
1257
|
+
else {
|
|
1258
|
+
sourcePath = authorizeExistingPath(kb.source_path, access);
|
|
1259
|
+
}
|
|
1260
|
+
const updateableKB = { ...kb, source_path: sourcePath };
|
|
813
1261
|
const activeUpdate = this.activeUpdates.get(updateableKB.id);
|
|
814
1262
|
if (activeUpdate) {
|
|
815
1263
|
onProgress?.(`Update already running for "${updateableKB.name}"; waiting for the active update.`);
|
|
816
1264
|
return activeUpdate;
|
|
817
1265
|
}
|
|
818
|
-
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal)).finally(() => {
|
|
1266
|
+
const updateRun = this.runExclusive(["global:mutation", `kb:${updateableKB.id}`], `Mutation for "${updateableKB.name}"`, () => this.runUpdate(updateableKB, onProgress, signal, access)).finally(() => {
|
|
819
1267
|
if (this.activeUpdates.get(updateableKB.id) === updateRun)
|
|
820
1268
|
this.activeUpdates.delete(updateableKB.id);
|
|
821
1269
|
});
|
|
822
1270
|
this.activeUpdates.set(updateableKB.id, updateRun);
|
|
823
1271
|
return updateRun;
|
|
824
1272
|
}
|
|
825
|
-
async runUpdate(kb, onProgress, signal) {
|
|
1273
|
+
async runUpdate(kb, onProgress, signal, access) {
|
|
826
1274
|
if (!this.db)
|
|
827
1275
|
throw new Error("Engine not initialized");
|
|
828
1276
|
throwIfAborted(signal);
|
|
829
|
-
const scanOptions = toScanOptions(parseAddOptions(kb.source_options));
|
|
1277
|
+
const scanOptions = toScanOptions(parseAddOptions(kb.source_options), access);
|
|
830
1278
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
831
1279
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
832
1280
|
updateKBStatus(this.db, kb.id, "indexing");
|
|
@@ -965,8 +1413,8 @@ export class KnowledgeEngine {
|
|
|
965
1413
|
stagedSymbols.push(...extractSymbols(chunks.map((chunk) => chunk.content).join("\n\n"), kb.source_path, "html"));
|
|
966
1414
|
await processChunks(chunks);
|
|
967
1415
|
}
|
|
968
|
-
else if (
|
|
969
|
-
const plan =
|
|
1416
|
+
else if (kb.source_type === "directory") {
|
|
1417
|
+
const plan = await planDirectoryScanAsync(kb.source_path, scanOptions, signal);
|
|
970
1418
|
plannedTotalFiles = plan.files;
|
|
971
1419
|
const planningMessage = `Planned directory scan: ${plan.files} files, ${formatBytes(plan.bytes)} scannable text, skipped ${plan.skippedTotal} (${plan.skippedSummary})`;
|
|
972
1420
|
updateIndexingJob(this.db, kb.id, {
|
|
@@ -977,12 +1425,27 @@ export class KnowledgeEngine {
|
|
|
977
1425
|
});
|
|
978
1426
|
onProgress?.(planningMessage);
|
|
979
1427
|
const skipped = createSkippedScanStats();
|
|
980
|
-
for (const file of
|
|
1428
|
+
for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptions, signal)) {
|
|
981
1429
|
if (signal?.aborted)
|
|
982
1430
|
throw new Error("Cancelled");
|
|
983
1431
|
const extracted = await extractScannableFileContentOrSkip(file, skipped, signal);
|
|
984
|
-
if (!extracted)
|
|
1432
|
+
if (!extracted) {
|
|
1433
|
+
for (const chunk of getChunksByFile(this.db, kb.id, file.relPath, 0, Number.MAX_SAFE_INTEGER)) {
|
|
1434
|
+
const entries = existingHashes.get(chunk.content_hash);
|
|
1435
|
+
const retainedIndex = entries?.findIndex((entry) => entry.id === chunk.id) ?? -1;
|
|
1436
|
+
if (!entries || retainedIndex < 0)
|
|
1437
|
+
continue;
|
|
1438
|
+
const [retained] = entries.splice(retainedIndex, 1);
|
|
1439
|
+
if (canReuseExistingVectors && retained) {
|
|
1440
|
+
const indexes = oldVectorIndexByHash.get(chunk.content_hash) ?? [];
|
|
1441
|
+
indexes.push(retained.vectorIndex);
|
|
1442
|
+
oldVectorIndexByHash.set(chunk.content_hash, indexes);
|
|
1443
|
+
}
|
|
1444
|
+
unchanged++;
|
|
1445
|
+
}
|
|
1446
|
+
stagedSymbols.push(...getSymbolsByFile(this.db, kb.id, file.relPath));
|
|
985
1447
|
continue;
|
|
1448
|
+
}
|
|
986
1449
|
const analysis = await analyzeIndexableContent(extracted.content, file.relPath, extracted.fileType);
|
|
987
1450
|
scannedFiles++;
|
|
988
1451
|
stagedSymbols.push(...analysis.symbols);
|
|
@@ -1018,7 +1481,7 @@ export class KnowledgeEngine {
|
|
|
1018
1481
|
onProgress?.(message);
|
|
1019
1482
|
}
|
|
1020
1483
|
else {
|
|
1021
|
-
const extracted = await extractSourceFileContent(kb.source_path, signal);
|
|
1484
|
+
const extracted = await extractSourceFileContent(kb.source_path, signal, undefined, access);
|
|
1022
1485
|
scannedFiles = 1;
|
|
1023
1486
|
const analysis = await analyzeIndexableContent(extracted.content, kb.source_path, extracted.fileType);
|
|
1024
1487
|
stagedSymbols.push(...analysis.symbols);
|
|
@@ -1125,6 +1588,7 @@ export class KnowledgeEngine {
|
|
|
1125
1588
|
updateKBEmbeddingMetadata(this.db, kb.id, embeddingModel, finalEmbeddingDimension === undefined ? null : embeddingSignature(embeddingConfig, finalEmbeddingDimension), finalEmbeddingDimension ?? null);
|
|
1126
1589
|
updateKBCounts(this.db, kb.id, finalChunkCount, getFileCount(this.db, kb.id));
|
|
1127
1590
|
updateKBSourceOptions(this.db, kb.id, serializeAddOptions(parseAddOptions(kb.source_options)));
|
|
1591
|
+
markKBIndexed(this.db, kb.id);
|
|
1128
1592
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1129
1593
|
const skippedReadySuffix = latestSkippedTotal > 0 ? `, skipped ${latestSkippedTotal} (${latestSkippedSummary})` : "";
|
|
1130
1594
|
const readyMessage = `Ready: +${addedCount} -${idsToRemove.length} =${unchanged}${skippedReadySuffix}`;
|
|
@@ -1163,43 +1627,82 @@ export class KnowledgeEngine {
|
|
|
1163
1627
|
const requestedMode = options.mode ?? "hybrid";
|
|
1164
1628
|
if (requestedMode === "auto") {
|
|
1165
1629
|
const primaryMode = chooseAutoMode(query);
|
|
1166
|
-
const
|
|
1630
|
+
const modes = [primaryMode, ...fallbackModesFor(primaryMode)];
|
|
1167
1631
|
const tried = [];
|
|
1632
|
+
const attemptSummaries = [];
|
|
1168
1633
|
const warnings = [];
|
|
1169
|
-
let
|
|
1170
|
-
|
|
1634
|
+
let lastMissResponse;
|
|
1635
|
+
let bestWeakResponse;
|
|
1636
|
+
for (const mode of modes) {
|
|
1171
1637
|
if (tried.includes(mode))
|
|
1172
1638
|
continue;
|
|
1173
1639
|
tried.push(mode);
|
|
1174
1640
|
throwIfAborted(signal);
|
|
1175
1641
|
const response = await this.search(query, { ...options, mode }, signal);
|
|
1176
|
-
|
|
1642
|
+
attemptSummaries.push(searchAttemptSummary(mode, response));
|
|
1177
1643
|
if (response.warnings)
|
|
1178
1644
|
warnings.push(...response.warnings);
|
|
1645
|
+
if (response.miss?.reason === "no_knowledge_bases" ||
|
|
1646
|
+
response.miss?.reason === "knowledge_base_not_found" ||
|
|
1647
|
+
response.miss?.reason === "no_searchable_knowledge_bases" ||
|
|
1648
|
+
response.miss?.reason === "no_indexed_chunks") {
|
|
1649
|
+
return {
|
|
1650
|
+
...response,
|
|
1651
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1652
|
+
mode_used: mode,
|
|
1653
|
+
retry_modes: tried.slice(0, -1),
|
|
1654
|
+
miss: { ...response.miss, attempts: attemptSummaries },
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1179
1657
|
if (!isWeakAutoResponse(query, response, primaryMode, mode)) {
|
|
1180
1658
|
return {
|
|
1181
1659
|
...response,
|
|
1182
1660
|
warnings: warnings.length > 0 ? [...new Set(warnings)] : response.warnings,
|
|
1183
1661
|
mode_used: mode,
|
|
1184
1662
|
retry_modes: tried.slice(0, -1),
|
|
1185
|
-
suggestions: response.suggestions,
|
|
1186
1663
|
};
|
|
1187
1664
|
}
|
|
1665
|
+
if (response.results.length > 0) {
|
|
1666
|
+
if (!bestWeakResponse || response.results.length > bestWeakResponse.response.results.length) {
|
|
1667
|
+
bestWeakResponse = { mode, response };
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
else {
|
|
1671
|
+
lastMissResponse = { mode, response };
|
|
1672
|
+
}
|
|
1188
1673
|
if (tried.length >= 3)
|
|
1189
1674
|
break;
|
|
1190
1675
|
}
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1676
|
+
if (lastMissResponse?.response.miss) {
|
|
1677
|
+
const attemptedModes = new Set(attemptSummaries.map((attempt) => attempt.mode));
|
|
1678
|
+
let action = lastMissResponse.response.miss.action;
|
|
1679
|
+
if (action === "try_semantic" && attemptedModes.has("semantic")) {
|
|
1680
|
+
action = attemptedModes.has("semantic_hybrid") ? "rephrase" : "try_semantic_hybrid";
|
|
1681
|
+
}
|
|
1682
|
+
else if (action === "try_semantic_hybrid" && attemptedModes.has("semantic_hybrid")) {
|
|
1683
|
+
action = "rephrase";
|
|
1684
|
+
}
|
|
1685
|
+
return {
|
|
1686
|
+
...lastMissResponse.response,
|
|
1687
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : lastMissResponse.response.warnings,
|
|
1688
|
+
mode_used: lastMissResponse.mode,
|
|
1689
|
+
retry_modes: tried.filter((mode) => mode !== lastMissResponse.mode),
|
|
1690
|
+
miss: {
|
|
1691
|
+
...lastMissResponse.response.miss,
|
|
1692
|
+
action,
|
|
1693
|
+
attempts: attemptSummaries,
|
|
1694
|
+
},
|
|
1695
|
+
};
|
|
1696
|
+
}
|
|
1697
|
+
if (bestWeakResponse) {
|
|
1698
|
+
return {
|
|
1699
|
+
...bestWeakResponse.response,
|
|
1700
|
+
warnings: warnings.length > 0 ? [...new Set(warnings)] : bestWeakResponse.response.warnings,
|
|
1701
|
+
mode_used: bestWeakResponse.mode,
|
|
1702
|
+
retry_modes: tried.filter((mode) => mode !== bestWeakResponse.mode),
|
|
1703
|
+
};
|
|
1704
|
+
}
|
|
1705
|
+
throw new Error("Auto search exhausted without a result or miss classification");
|
|
1203
1706
|
}
|
|
1204
1707
|
const db = this.db;
|
|
1205
1708
|
const { mode = "hybrid", offset: requestedOffset, kb_id, filters, diversity = "balanced" } = options;
|
|
@@ -1209,8 +1712,9 @@ export class KnowledgeEngine {
|
|
|
1209
1712
|
const queryTokens = tokenizeForSimilarity(query);
|
|
1210
1713
|
const normalizedFileType = normalizeFileTypeFilter(filters?.file_type);
|
|
1211
1714
|
const warnings = [];
|
|
1715
|
+
const allKBs = listKBs(db);
|
|
1212
1716
|
const selectedKB = kb_id ? (getKB(db, kb_id) ?? getKBByName(db, kb_id)) : undefined;
|
|
1213
|
-
const availableKBs = kb_id ? [selectedKB].filter(Boolean) :
|
|
1717
|
+
const availableKBs = kb_id ? [selectedKB].filter(Boolean) : allKBs;
|
|
1214
1718
|
const kbs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1215
1719
|
const kbById = new Map(kbs.map((kb) => [kb.id, kb]));
|
|
1216
1720
|
for (const kb of availableKBs) {
|
|
@@ -1236,11 +1740,30 @@ export class KnowledgeEngine {
|
|
|
1236
1740
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1237
1741
|
mode_used: mode,
|
|
1238
1742
|
tuning: tuningSummary,
|
|
1743
|
+
miss: classifySearchMiss({
|
|
1744
|
+
allKnowledgeBases: allKBs,
|
|
1745
|
+
availableKnowledgeBases: availableKBs,
|
|
1746
|
+
searchedKnowledgeBases: kbs,
|
|
1747
|
+
requestedKbId: kb_id,
|
|
1748
|
+
mode,
|
|
1749
|
+
fileType: filters?.file_type,
|
|
1750
|
+
pathPattern: filters?.path_pattern,
|
|
1751
|
+
minimumScore: tuning.minHybridScore,
|
|
1752
|
+
offset,
|
|
1753
|
+
retrievedCandidates: 0,
|
|
1754
|
+
thresholdCandidates: 0,
|
|
1755
|
+
eligibleCandidates: 0,
|
|
1756
|
+
filteredCandidates: 0,
|
|
1757
|
+
rankedCandidates: 0,
|
|
1758
|
+
totalResults: 0,
|
|
1759
|
+
embeddingIncompatibleCount: 0,
|
|
1760
|
+
}),
|
|
1239
1761
|
};
|
|
1240
1762
|
}
|
|
1241
1763
|
const allResults = [];
|
|
1242
1764
|
const vectorsByChunkId = new Map();
|
|
1243
1765
|
let queryEmbedding;
|
|
1766
|
+
const embeddingIncompatibleKbIds = new Set();
|
|
1244
1767
|
for (const kb of kbs) {
|
|
1245
1768
|
throwIfAborted(signal);
|
|
1246
1769
|
if (kb.chunk_count === 0)
|
|
@@ -1258,6 +1781,7 @@ export class KnowledgeEngine {
|
|
|
1258
1781
|
throwIfAborted(signal);
|
|
1259
1782
|
const compatibility = inspectEmbeddingCompatibility(kb.embedding_signature, kb.embedding_dimension, queryEmbeddingConfig, queryVec.length);
|
|
1260
1783
|
if (!compatibility.compatible) {
|
|
1784
|
+
embeddingIncompatibleKbIds.add(kb.id);
|
|
1261
1785
|
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1262
1786
|
continue;
|
|
1263
1787
|
}
|
|
@@ -1283,6 +1807,7 @@ export class KnowledgeEngine {
|
|
|
1283
1807
|
vectorsByChunkId.set(chunkId, vector);
|
|
1284
1808
|
}
|
|
1285
1809
|
else {
|
|
1810
|
+
embeddingIncompatibleKbIds.add(kb.id);
|
|
1286
1811
|
warnings.push(embeddingMismatchWarning(kb, compatibility));
|
|
1287
1812
|
}
|
|
1288
1813
|
const fused = weightedScoreFusion(bm25Results, vecResults);
|
|
@@ -1307,30 +1832,30 @@ export class KnowledgeEngine {
|
|
|
1307
1832
|
result.score = ranking.adjusted_score;
|
|
1308
1833
|
}
|
|
1309
1834
|
}
|
|
1310
|
-
let
|
|
1835
|
+
let thresholded = unique.filter((result) => rankingByChunkId.has(result.chunkId));
|
|
1836
|
+
if (retrievalMode !== "fast" && retrievalMode !== "semantic") {
|
|
1837
|
+
thresholded = thresholded.filter((result) => result.score >= tuning.minHybridScore);
|
|
1838
|
+
}
|
|
1839
|
+
const thresholdCandidates = thresholded.length;
|
|
1840
|
+
let eligible = thresholded;
|
|
1311
1841
|
if (retrievalMode === "fast") {
|
|
1312
|
-
|
|
1842
|
+
eligible = thresholded.filter((result) => {
|
|
1313
1843
|
const chunk = getChunkById(db, result.chunkId);
|
|
1314
|
-
|
|
1315
|
-
return false;
|
|
1316
|
-
return hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens);
|
|
1844
|
+
return chunk ? hasAnyLexicalEvidence(buildChunkEmbeddingText(chunk), queryTokens) : false;
|
|
1317
1845
|
});
|
|
1318
1846
|
}
|
|
1319
|
-
else if (retrievalMode !== "semantic") {
|
|
1320
|
-
|
|
1847
|
+
else if (retrievalMode !== "semantic" && retrievalMode !== "semantic_hybrid") {
|
|
1848
|
+
eligible = thresholded.filter((result) => {
|
|
1321
1849
|
const chunk = getChunkById(db, result.chunkId);
|
|
1322
|
-
|
|
1323
|
-
return false;
|
|
1324
|
-
if (retrievalMode === "semantic_hybrid")
|
|
1325
|
-
return result.score >= tuning.minHybridScore;
|
|
1326
|
-
return result.score >= tuning.minHybridScore && hasEnoughLexicalEvidence(chunk, queryTokens);
|
|
1850
|
+
return chunk ? hasEnoughLexicalEvidence(chunk, queryTokens) : false;
|
|
1327
1851
|
});
|
|
1328
1852
|
}
|
|
1329
|
-
|
|
1853
|
+
eligible.sort((a, b) => b.score - a.score);
|
|
1854
|
+
const eligibleCandidates = eligible.length;
|
|
1330
1855
|
// Apply metadata filters post-retrieval
|
|
1331
|
-
let filtered =
|
|
1856
|
+
let filtered = eligible;
|
|
1332
1857
|
if (normalizedFileType || filters?.path_pattern) {
|
|
1333
|
-
filtered =
|
|
1858
|
+
filtered = eligible.filter((r) => {
|
|
1334
1859
|
const chunk = getChunkById(db, r.chunkId);
|
|
1335
1860
|
if (!chunk)
|
|
1336
1861
|
return false;
|
|
@@ -1380,7 +1905,7 @@ export class KnowledgeEngine {
|
|
|
1380
1905
|
const page = diversified.slice(offset, offset + limit);
|
|
1381
1906
|
const results = page.map((r) => {
|
|
1382
1907
|
const kb = kbById.get(r.chunk.kb_id);
|
|
1383
|
-
const
|
|
1908
|
+
const stale = kb?.status === "stale";
|
|
1384
1909
|
return {
|
|
1385
1910
|
content: r.content,
|
|
1386
1911
|
file_path: r.chunk.file_path,
|
|
@@ -1395,26 +1920,48 @@ export class KnowledgeEngine {
|
|
|
1395
1920
|
chunk_id: r.chunk.id,
|
|
1396
1921
|
chunk_hash: r.chunk.content_hash,
|
|
1397
1922
|
indexed_at: r.chunk.indexed_at,
|
|
1398
|
-
|
|
1399
|
-
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
1923
|
+
stale,
|
|
1400
1924
|
match_reason: matchReasonFor(mode),
|
|
1401
1925
|
source_chunk_ids: r.sourceChunkIds,
|
|
1402
1926
|
},
|
|
1403
1927
|
};
|
|
1404
1928
|
});
|
|
1929
|
+
const deepTuning = {
|
|
1930
|
+
...tuningSummary,
|
|
1931
|
+
retrieved_candidates: unique.length,
|
|
1932
|
+
threshold_candidates: thresholdCandidates,
|
|
1933
|
+
eligible_candidates: eligibleCandidates,
|
|
1934
|
+
filtered_candidates: filtered.length,
|
|
1935
|
+
diversity_candidates: ranked.length,
|
|
1936
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
1937
|
+
};
|
|
1405
1938
|
return {
|
|
1406
1939
|
results,
|
|
1407
1940
|
total_count: diversified.length,
|
|
1408
1941
|
has_more: offset + limit < diversified.length,
|
|
1409
1942
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1410
1943
|
mode_used: mode,
|
|
1411
|
-
tuning:
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1944
|
+
tuning: deepTuning,
|
|
1945
|
+
miss: results.length === 0
|
|
1946
|
+
? classifySearchMiss({
|
|
1947
|
+
allKnowledgeBases: allKBs,
|
|
1948
|
+
availableKnowledgeBases: availableKBs,
|
|
1949
|
+
searchedKnowledgeBases: kbs,
|
|
1950
|
+
requestedKbId: kb_id,
|
|
1951
|
+
mode,
|
|
1952
|
+
fileType: filters?.file_type,
|
|
1953
|
+
pathPattern: filters?.path_pattern,
|
|
1954
|
+
minimumScore: tuning.minHybridScore,
|
|
1955
|
+
offset,
|
|
1956
|
+
retrievedCandidates: unique.length,
|
|
1957
|
+
thresholdCandidates,
|
|
1958
|
+
eligibleCandidates,
|
|
1959
|
+
filteredCandidates: filtered.length,
|
|
1960
|
+
rankedCandidates: ranked.length,
|
|
1961
|
+
totalResults: diversified.length,
|
|
1962
|
+
embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
|
|
1963
|
+
})
|
|
1964
|
+
: undefined,
|
|
1418
1965
|
};
|
|
1419
1966
|
}
|
|
1420
1967
|
const ranked = [];
|
|
@@ -1471,36 +2018,52 @@ export class KnowledgeEngine {
|
|
|
1471
2018
|
end_line: r.endLine,
|
|
1472
2019
|
provenance: (() => {
|
|
1473
2020
|
const kb = kbById.get(r.chunk.kb_id);
|
|
1474
|
-
const
|
|
2021
|
+
const stale = kb?.status === "stale";
|
|
1475
2022
|
return {
|
|
1476
2023
|
chunk_id: r.chunk.id,
|
|
1477
2024
|
chunk_hash: r.chunk.content_hash,
|
|
1478
2025
|
indexed_at: r.chunk.indexed_at,
|
|
1479
|
-
|
|
1480
|
-
stale: sourceMtime !== undefined ? sourceMtime > r.chunk.indexed_at : kb?.status === "stale",
|
|
2026
|
+
stale,
|
|
1481
2027
|
match_reason: matchReasonFor(mode),
|
|
1482
2028
|
source_chunk_ids: r.sourceChunkIds,
|
|
1483
2029
|
};
|
|
1484
2030
|
})(),
|
|
1485
2031
|
}));
|
|
2032
|
+
const responseTuning = {
|
|
2033
|
+
...tuningSummary,
|
|
2034
|
+
retrieved_candidates: unique.length,
|
|
2035
|
+
threshold_candidates: thresholdCandidates,
|
|
2036
|
+
eligible_candidates: eligibleCandidates,
|
|
2037
|
+
filtered_candidates: filtered.length,
|
|
2038
|
+
diversity_candidates: ranked.length,
|
|
2039
|
+
candidate_pool_truncated: candidatePoolTruncated,
|
|
2040
|
+
};
|
|
1486
2041
|
return {
|
|
1487
2042
|
results,
|
|
1488
2043
|
total_count: total,
|
|
1489
2044
|
has_more: offset + limit < total,
|
|
1490
2045
|
warnings: warnings.length > 0 ? warnings : undefined,
|
|
1491
2046
|
mode_used: mode,
|
|
1492
|
-
tuning:
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
2047
|
+
tuning: responseTuning,
|
|
2048
|
+
miss: results.length === 0
|
|
2049
|
+
? classifySearchMiss({
|
|
2050
|
+
allKnowledgeBases: allKBs,
|
|
2051
|
+
availableKnowledgeBases: availableKBs,
|
|
2052
|
+
searchedKnowledgeBases: kbs,
|
|
2053
|
+
requestedKbId: kb_id,
|
|
2054
|
+
mode,
|
|
2055
|
+
fileType: filters?.file_type,
|
|
2056
|
+
pathPattern: filters?.path_pattern,
|
|
2057
|
+
minimumScore: tuning.minHybridScore,
|
|
2058
|
+
offset,
|
|
2059
|
+
retrievedCandidates: unique.length,
|
|
2060
|
+
thresholdCandidates,
|
|
2061
|
+
eligibleCandidates,
|
|
2062
|
+
filteredCandidates: filtered.length,
|
|
2063
|
+
rankedCandidates: ranked.length,
|
|
2064
|
+
totalResults: total,
|
|
2065
|
+
embeddingIncompatibleCount: embeddingIncompatibleKbIds.size,
|
|
2066
|
+
})
|
|
1504
2067
|
: undefined,
|
|
1505
2068
|
};
|
|
1506
2069
|
}
|
|
@@ -1518,11 +2081,11 @@ export class KnowledgeEngine {
|
|
|
1518
2081
|
this.deleteVectorFile(kb.id);
|
|
1519
2082
|
return true;
|
|
1520
2083
|
}
|
|
1521
|
-
list(signal) {
|
|
2084
|
+
list(signal, limit) {
|
|
1522
2085
|
if (!this.db)
|
|
1523
2086
|
return [];
|
|
1524
2087
|
throwIfAborted(signal);
|
|
1525
|
-
return listKBs(this.db);
|
|
2088
|
+
return listKBs(this.db, limit);
|
|
1526
2089
|
}
|
|
1527
2090
|
clear() {
|
|
1528
2091
|
if (!this.db)
|
|
@@ -1546,16 +2109,32 @@ export class KnowledgeEngine {
|
|
|
1546
2109
|
return diagnoseKB(db, kb, signal);
|
|
1547
2110
|
});
|
|
1548
2111
|
}
|
|
1549
|
-
|
|
2112
|
+
async diagnoseAsync(signal, knowledgeBasesOverride, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
1550
2113
|
if (!this.db)
|
|
1551
|
-
return
|
|
2114
|
+
return [];
|
|
1552
2115
|
throwIfAborted(signal);
|
|
1553
2116
|
const db = this.db;
|
|
1554
|
-
const
|
|
1555
|
-
|
|
2117
|
+
const diagnostics = [];
|
|
2118
|
+
let remainingPaths = maxDiagnosticPaths;
|
|
2119
|
+
for (const kb of knowledgeBasesOverride ?? listKBs(db)) {
|
|
1556
2120
|
throwIfAborted(signal);
|
|
1557
|
-
|
|
1558
|
-
|
|
2121
|
+
const diagnostic = await diagnoseKBAsync(db, kb, signal, Math.max(0, remainingPaths));
|
|
2122
|
+
diagnostics.push(diagnostic);
|
|
2123
|
+
remainingPaths = Math.max(0, remainingPaths - diagnostic.stale_files.length - diagnostic.orphan_files.length);
|
|
2124
|
+
}
|
|
2125
|
+
return diagnostics;
|
|
2126
|
+
}
|
|
2127
|
+
statusSnapshot(signal, diagnosticsOverride, knowledgeBasesOverride) {
|
|
2128
|
+
if (!this.db)
|
|
2129
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
2130
|
+
throwIfAborted(signal);
|
|
2131
|
+
const db = this.db;
|
|
2132
|
+
const knowledgeBases = knowledgeBasesOverride ?? listKBs(db);
|
|
2133
|
+
const diagnostics = diagnosticsOverride ??
|
|
2134
|
+
knowledgeBases.map((kb) => {
|
|
2135
|
+
throwIfAborted(signal);
|
|
2136
|
+
return diagnoseKB(db, kb, signal);
|
|
2137
|
+
});
|
|
1559
2138
|
try {
|
|
1560
2139
|
const config = resolveEmbeddingConfig();
|
|
1561
2140
|
const activeModel = embeddingConfigLabel(config);
|
|
@@ -1586,16 +2165,46 @@ export class KnowledgeEngine {
|
|
|
1586
2165
|
};
|
|
1587
2166
|
}
|
|
1588
2167
|
}
|
|
2168
|
+
async statusSnapshotAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
2169
|
+
if (!this.db)
|
|
2170
|
+
return { knowledge_bases: [], diagnostics: [], embeddings: [] };
|
|
2171
|
+
const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
|
|
2172
|
+
if (knowledgeBases.length > maxKnowledgeBases) {
|
|
2173
|
+
throw new Error(`Knowledge status exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
|
|
2174
|
+
}
|
|
2175
|
+
return this.statusSnapshot(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
|
|
2176
|
+
}
|
|
1589
2177
|
symbolSearch(query, options = {}, signal) {
|
|
1590
2178
|
if (!this.db)
|
|
1591
2179
|
throw new Error("Engine not initialized");
|
|
1592
2180
|
throwIfAborted(signal);
|
|
1593
2181
|
const db = this.db;
|
|
2182
|
+
const allKBs = listKBs(db);
|
|
1594
2183
|
const selectedKB = options.kb_id ? (getKB(db, options.kb_id) ?? getKBByName(db, options.kb_id)) : undefined;
|
|
1595
|
-
|
|
1596
|
-
|
|
2184
|
+
const availableKBs = options.kb_id ? [selectedKB].filter(Boolean) : allKBs;
|
|
2185
|
+
const searchedKBs = availableKBs.filter((kb) => kb.status === "ready" || kb.status === "stale");
|
|
1597
2186
|
const limit = Math.max(1, Math.min(200, options.limit ?? 20));
|
|
1598
2187
|
const offset = Math.max(0, options.offset ?? 0);
|
|
2188
|
+
if (searchedKBs.length === 0) {
|
|
2189
|
+
return {
|
|
2190
|
+
results: [],
|
|
2191
|
+
total_count: 0,
|
|
2192
|
+
has_more: false,
|
|
2193
|
+
miss: classifySymbolSearchMiss({
|
|
2194
|
+
allKnowledgeBases: allKBs,
|
|
2195
|
+
availableKnowledgeBases: availableKBs,
|
|
2196
|
+
searchedKnowledgeBases: searchedKBs,
|
|
2197
|
+
requestedKbId: options.kb_id,
|
|
2198
|
+
kind: options.kind,
|
|
2199
|
+
filePattern: options.file_pattern,
|
|
2200
|
+
exact: options.exact === true,
|
|
2201
|
+
offset,
|
|
2202
|
+
totalResults: 0,
|
|
2203
|
+
indexedSymbols: 0,
|
|
2204
|
+
unfilteredMatches: 0,
|
|
2205
|
+
}),
|
|
2206
|
+
};
|
|
2207
|
+
}
|
|
1599
2208
|
const symbolOptions = {
|
|
1600
2209
|
kbId: selectedKB?.id,
|
|
1601
2210
|
kind: options.kind,
|
|
@@ -1611,32 +2220,55 @@ export class KnowledgeEngine {
|
|
|
1611
2220
|
const page = symbols.slice(0, limit);
|
|
1612
2221
|
const total = countSymbols(db, query, symbolOptions);
|
|
1613
2222
|
throwIfAborted(signal);
|
|
2223
|
+
const results = page.map((symbol) => {
|
|
2224
|
+
const kb = getKB(db, symbol.kb_id);
|
|
2225
|
+
return {
|
|
2226
|
+
name: symbol.name,
|
|
2227
|
+
kind: symbol.kind,
|
|
2228
|
+
file_path: symbol.file_path,
|
|
2229
|
+
file_type: symbol.file_type,
|
|
2230
|
+
kb_name: kb?.name ?? "unknown",
|
|
2231
|
+
start_line: symbol.start_line,
|
|
2232
|
+
end_line: symbol.end_line,
|
|
2233
|
+
signature: symbol.signature ?? undefined,
|
|
2234
|
+
container_name: symbol.container_name ?? undefined,
|
|
2235
|
+
text: symbol.text,
|
|
2236
|
+
indexed_at: symbol.indexed_at,
|
|
2237
|
+
};
|
|
2238
|
+
});
|
|
2239
|
+
let miss;
|
|
2240
|
+
if (results.length === 0) {
|
|
2241
|
+
const indexedSymbols = countSymbols(db, "", { kbId: selectedKB?.id });
|
|
2242
|
+
const unfilteredMatches = total === 0 && (options.kind || options.file_pattern)
|
|
2243
|
+
? countSymbols(db, query, { kbId: selectedKB?.id, exact: options.exact })
|
|
2244
|
+
: total;
|
|
2245
|
+
throwIfAborted(signal);
|
|
2246
|
+
miss = classifySymbolSearchMiss({
|
|
2247
|
+
allKnowledgeBases: allKBs,
|
|
2248
|
+
availableKnowledgeBases: availableKBs,
|
|
2249
|
+
searchedKnowledgeBases: searchedKBs,
|
|
2250
|
+
requestedKbId: options.kb_id,
|
|
2251
|
+
kind: options.kind,
|
|
2252
|
+
filePattern: options.file_pattern,
|
|
2253
|
+
exact: options.exact === true,
|
|
2254
|
+
offset,
|
|
2255
|
+
totalResults: total,
|
|
2256
|
+
indexedSymbols,
|
|
2257
|
+
unfilteredMatches,
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
1614
2260
|
return {
|
|
1615
|
-
results
|
|
1616
|
-
const kb = getKB(db, symbol.kb_id);
|
|
1617
|
-
return {
|
|
1618
|
-
name: symbol.name,
|
|
1619
|
-
kind: symbol.kind,
|
|
1620
|
-
file_path: symbol.file_path,
|
|
1621
|
-
file_type: symbol.file_type,
|
|
1622
|
-
kb_name: kb?.name ?? "unknown",
|
|
1623
|
-
start_line: symbol.start_line,
|
|
1624
|
-
end_line: symbol.end_line,
|
|
1625
|
-
signature: symbol.signature ?? undefined,
|
|
1626
|
-
container_name: symbol.container_name ?? undefined,
|
|
1627
|
-
text: symbol.text,
|
|
1628
|
-
indexed_at: symbol.indexed_at,
|
|
1629
|
-
};
|
|
1630
|
-
}),
|
|
2261
|
+
results,
|
|
1631
2262
|
total_count: total,
|
|
1632
2263
|
has_more: symbols.length > limit,
|
|
2264
|
+
miss,
|
|
1633
2265
|
};
|
|
1634
2266
|
}
|
|
1635
|
-
doctor(signal) {
|
|
2267
|
+
doctor(signal, diagnosticsOverride, knowledgeBasesOverride) {
|
|
1636
2268
|
throwIfAborted(signal);
|
|
1637
|
-
const diagnostics = this.diagnose(signal);
|
|
2269
|
+
const diagnostics = diagnosticsOverride ?? this.diagnose(signal);
|
|
1638
2270
|
const issues = [];
|
|
1639
|
-
const kbs = this.list();
|
|
2271
|
+
const kbs = knowledgeBasesOverride ?? this.list();
|
|
1640
2272
|
if (kbs.length === 0) {
|
|
1641
2273
|
issues.push({
|
|
1642
2274
|
severity: "blocking",
|
|
@@ -1666,7 +2298,16 @@ export class KnowledgeEngine {
|
|
|
1666
2298
|
action_code: "rebuild_kb",
|
|
1667
2299
|
});
|
|
1668
2300
|
}
|
|
1669
|
-
if (diagnostic.
|
|
2301
|
+
if (diagnostic.scan_error) {
|
|
2302
|
+
issues.push({
|
|
2303
|
+
severity: "warning",
|
|
2304
|
+
kb_name: diagnostic.kb_name,
|
|
2305
|
+
message: `Source diagnostics scan failed: ${diagnostic.scan_error}`,
|
|
2306
|
+
action: "Check source accessibility and scanner limits, then run knowledge_doctor again.",
|
|
2307
|
+
action_code: "check_source",
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
else if (diagnostic.coverage_percent < 80) {
|
|
1670
2311
|
issues.push({
|
|
1671
2312
|
severity: "warning",
|
|
1672
2313
|
kb_name: diagnostic.kb_name,
|
|
@@ -1675,20 +2316,22 @@ export class KnowledgeEngine {
|
|
|
1675
2316
|
action_code: "review_skipped_scope",
|
|
1676
2317
|
});
|
|
1677
2318
|
}
|
|
1678
|
-
|
|
2319
|
+
const staleFileCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
|
|
2320
|
+
if (staleFileCount > 0) {
|
|
1679
2321
|
issues.push({
|
|
1680
2322
|
severity: "warning",
|
|
1681
2323
|
kb_name: diagnostic.kb_name,
|
|
1682
|
-
message: `${
|
|
2324
|
+
message: `${staleFileCount} files changed after indexing.`,
|
|
1683
2325
|
action: "Run knowledge_update for this KB.",
|
|
1684
2326
|
action_code: "run_update",
|
|
1685
2327
|
});
|
|
1686
2328
|
}
|
|
1687
|
-
|
|
2329
|
+
const orphanFileCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
|
|
2330
|
+
if (orphanFileCount > 0) {
|
|
1688
2331
|
issues.push({
|
|
1689
2332
|
severity: "warning",
|
|
1690
2333
|
kb_name: diagnostic.kb_name,
|
|
1691
|
-
message: `${
|
|
2334
|
+
message: `${orphanFileCount} indexed files no longer exist in the source.`,
|
|
1692
2335
|
action: "Run knowledge_update or rebuild the KB.",
|
|
1693
2336
|
action_code: "run_update",
|
|
1694
2337
|
});
|
|
@@ -1797,23 +2440,68 @@ export class KnowledgeEngine {
|
|
|
1797
2440
|
})),
|
|
1798
2441
|
};
|
|
1799
2442
|
}
|
|
1800
|
-
async
|
|
2443
|
+
async doctorAsync(signal, maxKnowledgeBases = Number.POSITIVE_INFINITY, maxDiagnosticPaths = Number.POSITIVE_INFINITY) {
|
|
2444
|
+
if (!this.db)
|
|
2445
|
+
return this.doctor(signal, [], []);
|
|
2446
|
+
const knowledgeBases = listKBs(this.db, Number.isFinite(maxKnowledgeBases) ? maxKnowledgeBases + 1 : undefined);
|
|
2447
|
+
if (knowledgeBases.length > maxKnowledgeBases) {
|
|
2448
|
+
throw new Error(`Knowledge doctor exceeds the ${maxKnowledgeBases}-knowledge-base RPC limit`);
|
|
2449
|
+
}
|
|
2450
|
+
return this.doctor(signal, await this.diagnoseAsync(signal, knowledgeBases, maxDiagnosticPaths), knowledgeBases);
|
|
2451
|
+
}
|
|
2452
|
+
async exportKB(nameOrId, outputPath, signal, onProgress, access) {
|
|
1801
2453
|
if (!this.db)
|
|
1802
2454
|
throw new Error("Engine not initialized");
|
|
1803
2455
|
throwIfAborted(signal);
|
|
1804
2456
|
const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
|
|
1805
2457
|
if (!kb)
|
|
1806
2458
|
throw new Error(`Knowledge base not found: ${nameOrId}`);
|
|
1807
|
-
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress));
|
|
2459
|
+
return this.runExclusive(["global:mutation", `kb:${kb.id}`], `Operation for "${kb.name}"`, () => this.exportKBUnlocked(kb, outputPath, signal, onProgress, access));
|
|
1808
2460
|
}
|
|
1809
|
-
async exportKBUnlocked(kb, outputPath, signal, onProgress) {
|
|
2461
|
+
async exportKBUnlocked(kb, outputPath, signal, onProgress, access) {
|
|
1810
2462
|
if (!this.db)
|
|
1811
2463
|
throw new Error("Engine not initialized");
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
2464
|
+
if (kb.chunk_count > MAX_EXPORT_CHUNKS) {
|
|
2465
|
+
throw new Error(`Export exceeds the ${MAX_EXPORT_CHUNKS}-chunk limit`);
|
|
2466
|
+
}
|
|
2467
|
+
const target = prepareSafeNewFile(authorizeOutputPath(outputPath, access));
|
|
2468
|
+
let tempFd;
|
|
2469
|
+
let tempDevice;
|
|
2470
|
+
let tempInode;
|
|
2471
|
+
try {
|
|
2472
|
+
tempFd = openSync(target.tempPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
2473
|
+
const opened = fstatSync(tempFd);
|
|
2474
|
+
if (!opened.isFile())
|
|
2475
|
+
throw new Error(`Export target is not a regular file: ${target.tempPath}`);
|
|
2476
|
+
tempDevice = opened.dev;
|
|
2477
|
+
tempInode = opened.ino;
|
|
2478
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2479
|
+
assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
|
|
2480
|
+
}
|
|
2481
|
+
catch (error) {
|
|
2482
|
+
if (tempFd !== undefined)
|
|
2483
|
+
closeSync(tempFd);
|
|
2484
|
+
throw error;
|
|
2485
|
+
}
|
|
2486
|
+
const stream = createWriteStream("", {
|
|
2487
|
+
autoClose: false,
|
|
2488
|
+
encoding: "utf-8",
|
|
2489
|
+
fd: tempFd,
|
|
2490
|
+
});
|
|
1815
2491
|
const progressBatchSize = resolveIndexEmbedBatchSize();
|
|
1816
2492
|
let count = 0;
|
|
2493
|
+
let bytes = 0;
|
|
2494
|
+
const deadline = Date.now() + MAX_EXPORT_DURATION_MS;
|
|
2495
|
+
const writeBoundedLine = async (line) => {
|
|
2496
|
+
if (Date.now() > deadline)
|
|
2497
|
+
throw new Error(`Export exceeded ${MAX_EXPORT_DURATION_MS}ms`);
|
|
2498
|
+
const lineBytes = Buffer.byteLength(line) + 1;
|
|
2499
|
+
if (bytes + lineBytes > MAX_EXPORT_BYTES) {
|
|
2500
|
+
throw new Error(`Export exceeds the ${MAX_EXPORT_BYTES}-byte limit`);
|
|
2501
|
+
}
|
|
2502
|
+
bytes += lineBytes;
|
|
2503
|
+
await writeLine(stream, line);
|
|
2504
|
+
};
|
|
1817
2505
|
const header = JSON.stringify({
|
|
1818
2506
|
name: kb.name,
|
|
1819
2507
|
description: kb.description,
|
|
@@ -1821,48 +2509,130 @@ export class KnowledgeEngine {
|
|
|
1821
2509
|
chunk_count: kb.chunk_count,
|
|
1822
2510
|
});
|
|
1823
2511
|
try {
|
|
1824
|
-
await
|
|
2512
|
+
await writeBoundedLine(header);
|
|
1825
2513
|
for (const chunk of iterateChunksByKB(this.db, kb.id)) {
|
|
1826
2514
|
throwIfAborted(signal);
|
|
1827
|
-
|
|
2515
|
+
const line = JSON.stringify({
|
|
1828
2516
|
content: chunk.content,
|
|
1829
2517
|
file_path: chunk.file_path,
|
|
1830
2518
|
file_type: chunk.file_type,
|
|
1831
2519
|
start_line: chunk.start_line,
|
|
1832
2520
|
end_line: chunk.end_line,
|
|
1833
2521
|
metadata_json: chunk.metadata_json,
|
|
1834
|
-
})
|
|
2522
|
+
});
|
|
2523
|
+
await writeBoundedLine(line);
|
|
1835
2524
|
count++;
|
|
1836
2525
|
if (count % progressBatchSize === 0)
|
|
1837
2526
|
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks...`);
|
|
1838
2527
|
}
|
|
1839
2528
|
await finishWriteStream(stream);
|
|
1840
2529
|
throwIfAborted(signal);
|
|
1841
|
-
|
|
2530
|
+
fsyncSync(tempFd);
|
|
2531
|
+
if (stream.fd === tempFd)
|
|
2532
|
+
stream.fd = null;
|
|
2533
|
+
closeSync(tempFd);
|
|
2534
|
+
tempFd = undefined;
|
|
2535
|
+
stream.destroy();
|
|
2536
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2537
|
+
assertOutputPathAuthorized(target.outputPath, target.parentDevice, target.parentInode, access);
|
|
2538
|
+
publishNewFile(target.tempPath, target);
|
|
1842
2539
|
onProgress?.(`Exported ${count}/${kb.chunk_count} chunks.`);
|
|
1843
2540
|
return count;
|
|
1844
2541
|
}
|
|
1845
2542
|
catch (error) {
|
|
2543
|
+
if (tempFd !== undefined) {
|
|
2544
|
+
const descriptor = tempFd;
|
|
2545
|
+
tempFd = undefined;
|
|
2546
|
+
if (stream.fd === descriptor)
|
|
2547
|
+
stream.fd = null;
|
|
2548
|
+
try {
|
|
2549
|
+
closeSync(descriptor);
|
|
2550
|
+
}
|
|
2551
|
+
catch (closeError) {
|
|
2552
|
+
if (closeError.code !== "EBADF") {
|
|
2553
|
+
console.error(`Failed to close export descriptor: ${String(closeError)}`);
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
1846
2557
|
stream.destroy();
|
|
1847
|
-
|
|
2558
|
+
try {
|
|
2559
|
+
const current = lstatSync(target.tempPath);
|
|
2560
|
+
if (!current.isSymbolicLink() && current.dev === tempDevice && current.ino === tempInode) {
|
|
2561
|
+
assertOpenedPathAuthorized(target.tempPath, tempDevice, tempInode, access);
|
|
2562
|
+
rmSync(target.tempPath, { force: true });
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2565
|
+
catch {
|
|
2566
|
+
// Leave an unverified path untouched rather than deleting outside the authorized root.
|
|
2567
|
+
}
|
|
1848
2568
|
throw error;
|
|
1849
2569
|
}
|
|
1850
2570
|
}
|
|
1851
|
-
async importKB(inputPath, onProgress, signal) {
|
|
2571
|
+
async importKB(inputPath, onProgress, signal, access) {
|
|
1852
2572
|
if (!this.db)
|
|
1853
2573
|
throw new Error("Engine not initialized");
|
|
1854
|
-
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal));
|
|
2574
|
+
return this.runExclusive("global:mutation", "Knowledge-base import", () => this.importKBUnlocked(inputPath, onProgress, signal, access));
|
|
1855
2575
|
}
|
|
1856
|
-
async importKBUnlocked(inputPath, onProgress, signal) {
|
|
2576
|
+
async importKBUnlocked(inputPath, onProgress, signal, access) {
|
|
1857
2577
|
if (!this.db)
|
|
1858
2578
|
throw new Error("Engine not initialized");
|
|
1859
2579
|
throwIfAborted(signal);
|
|
1860
|
-
const { createReadStream } = await import("node:fs");
|
|
1861
|
-
const { createInterface } = await import("node:readline");
|
|
1862
2580
|
const embeddingConfig = resolveEmbeddingConfig();
|
|
1863
2581
|
const embeddingBatchSize = resolveIndexEmbedBatchSize();
|
|
1864
|
-
|
|
2582
|
+
inputPath = authorizeExistingPath(inputPath, access);
|
|
2583
|
+
const directInput = lstatSync(inputPath);
|
|
2584
|
+
if (directInput.isSymbolicLink() || !directInput.isFile()) {
|
|
2585
|
+
throw new Error(`Import source is not a regular non-symlink file: ${inputPath}`);
|
|
2586
|
+
}
|
|
2587
|
+
const inputFd = openSync(inputPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
2588
|
+
const inputInfo = fstatSync(inputFd);
|
|
2589
|
+
if (inputInfo.dev !== directInput.dev || inputInfo.ino !== directInput.ino) {
|
|
2590
|
+
closeSync(inputFd);
|
|
2591
|
+
throw new Error(`Import source changed while opening: ${inputPath}`);
|
|
2592
|
+
}
|
|
2593
|
+
if (!inputInfo.isFile()) {
|
|
2594
|
+
closeSync(inputFd);
|
|
2595
|
+
throw new Error(`Import source is not a regular file: ${inputPath}`);
|
|
2596
|
+
}
|
|
2597
|
+
try {
|
|
2598
|
+
assertOpenedPathAuthorized(inputPath, inputInfo.dev, inputInfo.ino, access);
|
|
2599
|
+
}
|
|
2600
|
+
catch (error) {
|
|
2601
|
+
closeSync(inputFd);
|
|
2602
|
+
throw error;
|
|
2603
|
+
}
|
|
2604
|
+
if (inputInfo.size > MAX_IMPORT_BYTES) {
|
|
2605
|
+
closeSync(inputFd);
|
|
2606
|
+
throw new Error(`Import file exceeds the ${MAX_IMPORT_BYTES}-byte limit: ${inputPath}`);
|
|
2607
|
+
}
|
|
2608
|
+
const stream = createReadStream("", { autoClose: true, encoding: "utf-8", fd: inputFd });
|
|
1865
2609
|
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
2610
|
+
let observedBytes = 0;
|
|
2611
|
+
let observedLineBytes = 0;
|
|
2612
|
+
stream.on("data", (chunk) => {
|
|
2613
|
+
observedBytes += Buffer.byteLength(String(chunk));
|
|
2614
|
+
if (observedBytes > MAX_IMPORT_BYTES) {
|
|
2615
|
+
stream.destroy(new Error(`Import exceeds the ${MAX_IMPORT_BYTES}-byte limit`));
|
|
2616
|
+
return;
|
|
2617
|
+
}
|
|
2618
|
+
const parts = String(chunk).split("\n");
|
|
2619
|
+
if (parts.length === 1) {
|
|
2620
|
+
observedLineBytes += Buffer.byteLength(parts[0] ?? "");
|
|
2621
|
+
}
|
|
2622
|
+
else {
|
|
2623
|
+
observedLineBytes += Buffer.byteLength(parts[0] ?? "");
|
|
2624
|
+
for (let index = 1; index < parts.length - 1; index++) {
|
|
2625
|
+
if (Buffer.byteLength(parts[index] ?? "") > MAX_IMPORT_LINE_BYTES) {
|
|
2626
|
+
stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
|
|
2627
|
+
return;
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
observedLineBytes = Buffer.byteLength(parts.at(-1) ?? "");
|
|
2631
|
+
}
|
|
2632
|
+
if (observedLineBytes > MAX_IMPORT_LINE_BYTES) {
|
|
2633
|
+
stream.destroy(new Error(`Import line exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`));
|
|
2634
|
+
}
|
|
2635
|
+
});
|
|
1866
2636
|
let header;
|
|
1867
2637
|
let kb;
|
|
1868
2638
|
let vectorWriter;
|
|
@@ -1908,13 +2678,28 @@ export class KnowledgeEngine {
|
|
|
1908
2678
|
for await (const rawLine of lines) {
|
|
1909
2679
|
throwIfAborted(signal);
|
|
1910
2680
|
lineNumber++;
|
|
2681
|
+
if (lineNumber > MAX_IMPORT_LINES)
|
|
2682
|
+
throw new Error(`Import exceeds the ${MAX_IMPORT_LINES}-line limit`);
|
|
1911
2683
|
const line = rawLine.trim();
|
|
2684
|
+
if (Buffer.byteLength(rawLine) > MAX_IMPORT_LINE_BYTES) {
|
|
2685
|
+
throw new Error(`Import line ${lineNumber} exceeds the ${MAX_IMPORT_LINE_BYTES}-byte limit`);
|
|
2686
|
+
}
|
|
1912
2687
|
if (!line)
|
|
1913
2688
|
continue;
|
|
1914
2689
|
if (!header) {
|
|
1915
|
-
|
|
1916
|
-
if (
|
|
1917
|
-
throw new Error("Import header
|
|
2690
|
+
const parsedHeader = JSON.parse(line);
|
|
2691
|
+
if (typeof parsedHeader !== "object" || parsedHeader === null || Array.isArray(parsedHeader)) {
|
|
2692
|
+
throw new Error("Import header must be an object");
|
|
2693
|
+
}
|
|
2694
|
+
const headerRecord = parsedHeader;
|
|
2695
|
+
header = {
|
|
2696
|
+
...validateKnowledgeBaseMetadata(headerRecord.name, headerRecord.description),
|
|
2697
|
+
chunk_count: headerRecord.chunk_count === undefined ? undefined : headerRecord.chunk_count,
|
|
2698
|
+
};
|
|
2699
|
+
if (header.chunk_count !== undefined &&
|
|
2700
|
+
(!Number.isInteger(header.chunk_count) || header.chunk_count < 0 || header.chunk_count > MAX_IMPORT_CHUNKS)) {
|
|
2701
|
+
throw new Error(`Import header chunk_count must be between 0 and ${MAX_IMPORT_CHUNKS}`);
|
|
2702
|
+
}
|
|
1918
2703
|
const existingKB = getKBByName(this.db, header.name);
|
|
1919
2704
|
if (existingKB) {
|
|
1920
2705
|
throw new Error(`Knowledge base "${header.name}" already exists. Use knowledge_update to refresh it, or knowledge_remove before importing a replacement.`);
|
|
@@ -1940,11 +2725,14 @@ export class KnowledgeEngine {
|
|
|
1940
2725
|
vectorWriter = openVectorWriter(tempVectorFile);
|
|
1941
2726
|
continue;
|
|
1942
2727
|
}
|
|
1943
|
-
const imported = JSON.parse(line);
|
|
2728
|
+
const imported = parseImportedChunk(JSON.parse(line));
|
|
1944
2729
|
const chunk = importedChunkToInsert(imported);
|
|
1945
2730
|
chunk.content_tokenized = preTokenizeForFTS(buildChunkEmbeddingText(chunk));
|
|
1946
2731
|
const finalizedChunks = await finalizeChunksForEmbedding([chunk], embeddingConfig, signal);
|
|
1947
2732
|
pendingChunks.push(...finalizedChunks);
|
|
2733
|
+
if (inserted + pendingChunks.length > MAX_IMPORT_CHUNKS) {
|
|
2734
|
+
throw new Error(`Import exceeds the ${MAX_IMPORT_CHUNKS}-chunk limit`);
|
|
2735
|
+
}
|
|
1948
2736
|
pendingSymbols.push(...extractSymbols(imported.content, imported.file_path, imported.file_type));
|
|
1949
2737
|
while (pendingChunks.length >= embeddingBatchSize)
|
|
1950
2738
|
await flushPending();
|
|
@@ -1959,6 +2747,7 @@ export class KnowledgeEngine {
|
|
|
1959
2747
|
renameSync(tempVectorFile, this.vectorPathFor(kb.id));
|
|
1960
2748
|
tempVectorFile = undefined;
|
|
1961
2749
|
updateKBCounts(this.db, kb.id, inserted, getFileCount(this.db, kb.id));
|
|
2750
|
+
markKBIndexed(this.db, kb.id);
|
|
1962
2751
|
updateKBStatus(this.db, kb.id, "ready");
|
|
1963
2752
|
const savedKB = getKB(this.db, kb.id);
|
|
1964
2753
|
if (!savedKB)
|