pi-knowledge 0.15.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +61 -23
- package/dist/index.d.ts +10 -4
- package/dist/index.js +145 -166
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +13 -8
- package/dist/src/engine.js +569 -100
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +12 -0
- package/dist/src/indexer/chunker.js +307 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +183 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +67 -0
- package/docs/configuration.md +40 -9
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { existsSync, statSync } from "node:fs";
|
|
2
|
+
import { stat as statFile } from "node:fs/promises";
|
|
2
3
|
import { join } from "node:path";
|
|
3
|
-
import {
|
|
4
|
+
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
|
|
5
|
+
import { createSkippedScanStats, iterateScannableFiles, iterateScannableFilesAsync, } from "../indexer/chunker.js";
|
|
4
6
|
import { getIndexingJob, iterateChunksByKB } from "../storage/sqlite.js";
|
|
5
7
|
const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
|
|
8
|
+
const MAX_DIAGNOSTIC_PATHS = 1_000;
|
|
9
|
+
const DIAGNOSTIC_YIELD_ITEMS = 1_000;
|
|
10
|
+
const MAX_DIAGNOSTIC_WORKING_PATHS = 100_000;
|
|
11
|
+
const MAX_DIAGNOSTIC_WORKING_PATH_BYTES = 64 * 1024 * 1024;
|
|
6
12
|
function staleIndexingMs() {
|
|
7
13
|
const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
|
|
8
14
|
return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
|
|
@@ -30,6 +36,10 @@ function throwIfAborted(signal) {
|
|
|
30
36
|
if (signal?.aborted)
|
|
31
37
|
throw new Error("Cancelled");
|
|
32
38
|
}
|
|
39
|
+
function diagnosticScanError(error) {
|
|
40
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
41
|
+
return message.length > 512 ? `${message.slice(0, 512)}…` : message;
|
|
42
|
+
}
|
|
33
43
|
export function diagnoseKB(db, kb, signal) {
|
|
34
44
|
const statusAgeMs = Date.now() - kb.updated_at;
|
|
35
45
|
const job = getIndexingJob(db, kb.id);
|
|
@@ -43,6 +53,10 @@ export function diagnoseKB(db, kb, signal) {
|
|
|
43
53
|
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
44
54
|
stale_files: [],
|
|
45
55
|
orphan_files: [],
|
|
56
|
+
stale_file_count: 0,
|
|
57
|
+
orphan_file_count: 0,
|
|
58
|
+
stale_files_truncated: false,
|
|
59
|
+
orphan_files_truncated: false,
|
|
46
60
|
coverage_percent: 100,
|
|
47
61
|
total_source_files: 0,
|
|
48
62
|
indexed_files: kb.file_count,
|
|
@@ -69,9 +83,11 @@ export function diagnoseKB(db, kb, signal) {
|
|
|
69
83
|
currentFiles.add(kb.source_path);
|
|
70
84
|
}
|
|
71
85
|
}
|
|
72
|
-
catch {
|
|
86
|
+
catch (error) {
|
|
73
87
|
if (signal?.aborted)
|
|
74
88
|
throw new Error("Cancelled");
|
|
89
|
+
result.coverage_percent = 0;
|
|
90
|
+
result.scan_error = diagnosticScanError(error);
|
|
75
91
|
return result;
|
|
76
92
|
}
|
|
77
93
|
result.total_source_files = currentFiles.size;
|
|
@@ -90,6 +106,7 @@ export function diagnoseKB(db, kb, signal) {
|
|
|
90
106
|
throwIfAborted(signal);
|
|
91
107
|
if (!currentFiles.has(filePath)) {
|
|
92
108
|
result.orphan_files.push(filePath);
|
|
109
|
+
result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
|
|
93
110
|
}
|
|
94
111
|
}
|
|
95
112
|
// Staleness detection: source files modified after last indexing
|
|
@@ -101,11 +118,127 @@ export function diagnoseKB(db, kb, signal) {
|
|
|
101
118
|
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
102
119
|
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
103
120
|
result.stale_files.push(relPath);
|
|
121
|
+
result.stale_file_count = (result.stale_file_count ?? 0) + 1;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
/* file unreadable — skip */
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return result;
|
|
129
|
+
}
|
|
130
|
+
export async function diagnoseKBAsync(db, kb, signal, maxDiagnosticPaths = MAX_DIAGNOSTIC_PATHS) {
|
|
131
|
+
const statusAgeMs = Date.now() - kb.updated_at;
|
|
132
|
+
const job = getIndexingJob(db, kb.id);
|
|
133
|
+
const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
|
|
134
|
+
const result = {
|
|
135
|
+
kb_id: kb.id,
|
|
136
|
+
kb_name: kb.name,
|
|
137
|
+
status: kb.status,
|
|
138
|
+
status_age_ms: statusAgeMs,
|
|
139
|
+
last_progress_age_ms: lastProgressAgeMs,
|
|
140
|
+
stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
|
|
141
|
+
stale_files: [],
|
|
142
|
+
orphan_files: [],
|
|
143
|
+
stale_file_count: 0,
|
|
144
|
+
orphan_file_count: 0,
|
|
145
|
+
stale_files_truncated: false,
|
|
146
|
+
orphan_files_truncated: false,
|
|
147
|
+
coverage_percent: 100,
|
|
148
|
+
total_source_files: 0,
|
|
149
|
+
indexed_files: kb.file_count,
|
|
150
|
+
skipped_files: createSkippedScanStats(),
|
|
151
|
+
job,
|
|
152
|
+
};
|
|
153
|
+
throwIfAborted(signal);
|
|
154
|
+
if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path))
|
|
155
|
+
return result;
|
|
156
|
+
const currentFiles = new Set();
|
|
157
|
+
let currentPathBytes = 0;
|
|
158
|
+
let isDirectory;
|
|
159
|
+
try {
|
|
160
|
+
isDirectory = (await statFile(kb.source_path)).isDirectory();
|
|
161
|
+
if (isDirectory) {
|
|
162
|
+
const skipped = createSkippedScanStats();
|
|
163
|
+
for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options), signal)) {
|
|
164
|
+
currentPathBytes += Buffer.byteLength(file.relPath);
|
|
165
|
+
if (currentFiles.size >= MAX_DIAGNOSTIC_WORKING_PATHS || currentPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
|
|
166
|
+
throw new Error("Diagnostic source path working-set limit exceeded");
|
|
167
|
+
}
|
|
168
|
+
currentFiles.add(file.relPath);
|
|
169
|
+
}
|
|
170
|
+
result.skipped_files = skipped;
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
currentFiles.add(kb.source_path);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
throwIfAborted(signal);
|
|
178
|
+
result.coverage_percent = 0;
|
|
179
|
+
result.scan_error = diagnosticScanError(error);
|
|
180
|
+
return result;
|
|
181
|
+
}
|
|
182
|
+
result.total_source_files = currentFiles.size;
|
|
183
|
+
result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
|
|
184
|
+
const latestIndexedByFile = new Map();
|
|
185
|
+
let indexedPathBytes = 0;
|
|
186
|
+
let processed = 0;
|
|
187
|
+
for (const chunk of iterateChunksByKB(db, kb.id)) {
|
|
188
|
+
throwIfAborted(signal);
|
|
189
|
+
if (!latestIndexedByFile.has(chunk.file_path)) {
|
|
190
|
+
indexedPathBytes += Buffer.byteLength(chunk.file_path);
|
|
191
|
+
if (latestIndexedByFile.size >= MAX_DIAGNOSTIC_WORKING_PATHS ||
|
|
192
|
+
indexedPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
|
|
193
|
+
result.scan_error = "Diagnostic indexed path working-set limit exceeded";
|
|
194
|
+
result.orphan_files_truncated = true;
|
|
195
|
+
result.stale_files_truncated = true;
|
|
196
|
+
return result;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
|
|
200
|
+
if (chunk.indexed_at > currentLatest)
|
|
201
|
+
latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
|
|
202
|
+
processed++;
|
|
203
|
+
if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
|
|
204
|
+
await yieldToEventLoop();
|
|
205
|
+
}
|
|
206
|
+
let remainingDiagnosticPaths = maxDiagnosticPaths;
|
|
207
|
+
for (const filePath of latestIndexedByFile.keys()) {
|
|
208
|
+
throwIfAborted(signal);
|
|
209
|
+
if (!currentFiles.has(filePath)) {
|
|
210
|
+
result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
|
|
211
|
+
if (remainingDiagnosticPaths > 0) {
|
|
212
|
+
result.orphan_files.push(filePath);
|
|
213
|
+
remainingDiagnosticPaths--;
|
|
214
|
+
}
|
|
215
|
+
else
|
|
216
|
+
result.orphan_files_truncated = true;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
processed = 0;
|
|
220
|
+
for (const relPath of currentFiles) {
|
|
221
|
+
throwIfAborted(signal);
|
|
222
|
+
const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
|
|
223
|
+
try {
|
|
224
|
+
const mtime = (await statFile(absPath)).mtimeMs;
|
|
225
|
+
const latestIndexed = latestIndexedByFile.get(relPath);
|
|
226
|
+
if (latestIndexed !== undefined && mtime > latestIndexed) {
|
|
227
|
+
result.stale_file_count = (result.stale_file_count ?? 0) + 1;
|
|
228
|
+
if (remainingDiagnosticPaths > 0) {
|
|
229
|
+
result.stale_files.push(relPath);
|
|
230
|
+
remainingDiagnosticPaths--;
|
|
231
|
+
}
|
|
232
|
+
else
|
|
233
|
+
result.stale_files_truncated = true;
|
|
104
234
|
}
|
|
105
235
|
}
|
|
106
236
|
catch {
|
|
107
237
|
/* file unreadable — skip */
|
|
108
238
|
}
|
|
239
|
+
processed++;
|
|
240
|
+
if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
|
|
241
|
+
await yieldToEventLoop();
|
|
109
242
|
}
|
|
110
243
|
return result;
|
|
111
244
|
}
|
package/dist/src/engine.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type DiagnosticResult } from "./diagnostics/health.ts";
|
|
2
2
|
import type { EmbeddingCompatibility } from "./embedding/provider.ts";
|
|
3
|
+
import { type FilesystemAccessPolicy } from "./filesystem-access.ts";
|
|
3
4
|
import { createSkippedScanStats } from "./indexer/chunker.ts";
|
|
4
5
|
import { type RankingDiagnostics } from "./search/ranking.ts";
|
|
5
6
|
import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
|
|
@@ -181,13 +182,14 @@ export declare class KnowledgeEngine {
|
|
|
181
182
|
initialize(knowledgeDir: string): Promise<void>;
|
|
182
183
|
private vectorPathFor;
|
|
183
184
|
private deleteVectorFile;
|
|
184
|
-
plan(source: string, options?: AddOptions, signal?: AbortSignal): IndexPlan;
|
|
185
|
-
|
|
185
|
+
plan(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): IndexPlan;
|
|
186
|
+
planAsync(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<IndexPlan>;
|
|
187
|
+
add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions, access?: FilesystemAccessPolicy): Promise<{
|
|
186
188
|
kb: KnowledgeBase;
|
|
187
189
|
chunkCount: number;
|
|
188
190
|
}>;
|
|
189
191
|
private addUnlocked;
|
|
190
|
-
update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
192
|
+
update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
|
|
191
193
|
added: number;
|
|
192
194
|
removed: number;
|
|
193
195
|
unchanged: number;
|
|
@@ -195,15 +197,18 @@ export declare class KnowledgeEngine {
|
|
|
195
197
|
private runUpdate;
|
|
196
198
|
search(query: string, options?: SearchOptions, signal?: AbortSignal): Promise<SearchResponse>;
|
|
197
199
|
remove(nameOrId: string): boolean;
|
|
198
|
-
list(signal?: AbortSignal): KnowledgeBase[];
|
|
200
|
+
list(signal?: AbortSignal, limit?: number): KnowledgeBase[];
|
|
199
201
|
clear(): void;
|
|
200
202
|
diagnose(signal?: AbortSignal): DiagnosticResult[];
|
|
201
|
-
|
|
203
|
+
diagnoseAsync(signal?: AbortSignal, knowledgeBasesOverride?: KnowledgeBase[], maxDiagnosticPaths?: number): Promise<DiagnosticResult[]>;
|
|
204
|
+
statusSnapshot(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): KnowledgeStatusSnapshot;
|
|
205
|
+
statusSnapshotAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<KnowledgeStatusSnapshot>;
|
|
202
206
|
symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
|
|
203
|
-
doctor(signal?: AbortSignal): DoctorReport;
|
|
204
|
-
|
|
207
|
+
doctor(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): DoctorReport;
|
|
208
|
+
doctorAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<DoctorReport>;
|
|
209
|
+
exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback, access?: FilesystemAccessPolicy): Promise<number>;
|
|
205
210
|
private exportKBUnlocked;
|
|
206
|
-
importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
|
|
211
|
+
importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
|
|
207
212
|
kb: KnowledgeBase;
|
|
208
213
|
chunkCount: number;
|
|
209
214
|
}>;
|