pi-knowledge 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +59 -22
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +145 -166
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +13 -8
  22. package/dist/src/engine.js +569 -100
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +10 -0
  26. package/dist/src/indexer/chunker.js +284 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/secure-files.d.ts +17 -0
  47. package/dist/src/secure-files.js +184 -0
  48. package/dist/src/status.js +12 -7
  49. package/dist/src/storage/sqlite.d.ts +4 -1
  50. package/dist/src/storage/sqlite.js +46 -3
  51. package/dist/src/type-guards.d.ts +1 -0
  52. package/dist/src/type-guards.js +3 -0
  53. package/dist/src/watcher/file-watcher.d.ts +3 -1
  54. package/dist/src/watcher/file-watcher.js +166 -32
  55. package/dist/src/windows-security.d.ts +4 -0
  56. package/dist/src/windows-security.js +47 -0
  57. package/docs/configuration.md +39 -9
  58. package/docs/daemon-profiles.md +302 -0
  59. package/docs/mcp-clients.md +266 -0
  60. package/docs/offline-mode.md +6 -6
  61. package/package.json +28 -2
@@ -1,8 +1,14 @@
1
1
  import { existsSync, statSync } from "node:fs";
2
+ import { stat as statFile } from "node:fs/promises";
2
3
  import { join } from "node:path";
3
- import { createSkippedScanStats, iterateScannableFiles, } from "../indexer/chunker.js";
4
+ import { setImmediate as yieldToEventLoop } from "node:timers/promises";
5
+ import { createSkippedScanStats, iterateScannableFiles, iterateScannableFilesAsync, } from "../indexer/chunker.js";
4
6
  import { getIndexingJob, iterateChunksByKB } from "../storage/sqlite.js";
5
7
  const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
8
+ const MAX_DIAGNOSTIC_PATHS = 1_000;
9
+ const DIAGNOSTIC_YIELD_ITEMS = 1_000;
10
+ const MAX_DIAGNOSTIC_WORKING_PATHS = 100_000;
11
+ const MAX_DIAGNOSTIC_WORKING_PATH_BYTES = 64 * 1024 * 1024;
6
12
  function staleIndexingMs() {
7
13
  const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
8
14
  return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
@@ -30,6 +36,10 @@ function throwIfAborted(signal) {
30
36
  if (signal?.aborted)
31
37
  throw new Error("Cancelled");
32
38
  }
39
+ function diagnosticScanError(error) {
40
+ const message = error instanceof Error ? error.message : String(error);
41
+ return message.length > 512 ? `${message.slice(0, 512)}…` : message;
42
+ }
33
43
  export function diagnoseKB(db, kb, signal) {
34
44
  const statusAgeMs = Date.now() - kb.updated_at;
35
45
  const job = getIndexingJob(db, kb.id);
@@ -43,6 +53,10 @@ export function diagnoseKB(db, kb, signal) {
43
53
  stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
44
54
  stale_files: [],
45
55
  orphan_files: [],
56
+ stale_file_count: 0,
57
+ orphan_file_count: 0,
58
+ stale_files_truncated: false,
59
+ orphan_files_truncated: false,
46
60
  coverage_percent: 100,
47
61
  total_source_files: 0,
48
62
  indexed_files: kb.file_count,
@@ -69,9 +83,11 @@ export function diagnoseKB(db, kb, signal) {
69
83
  currentFiles.add(kb.source_path);
70
84
  }
71
85
  }
72
- catch {
86
+ catch (error) {
73
87
  if (signal?.aborted)
74
88
  throw new Error("Cancelled");
89
+ result.coverage_percent = 0;
90
+ result.scan_error = diagnosticScanError(error);
75
91
  return result;
76
92
  }
77
93
  result.total_source_files = currentFiles.size;
@@ -90,6 +106,7 @@ export function diagnoseKB(db, kb, signal) {
90
106
  throwIfAborted(signal);
91
107
  if (!currentFiles.has(filePath)) {
92
108
  result.orphan_files.push(filePath);
109
+ result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
93
110
  }
94
111
  }
95
112
  // Staleness detection: source files modified after last indexing
@@ -101,11 +118,127 @@ export function diagnoseKB(db, kb, signal) {
101
118
  const latestIndexed = latestIndexedByFile.get(relPath);
102
119
  if (latestIndexed !== undefined && mtime > latestIndexed) {
103
120
  result.stale_files.push(relPath);
121
+ result.stale_file_count = (result.stale_file_count ?? 0) + 1;
122
+ }
123
+ }
124
+ catch {
125
+ /* file unreadable — skip */
126
+ }
127
+ }
128
+ return result;
129
+ }
130
+ export async function diagnoseKBAsync(db, kb, signal, maxDiagnosticPaths = MAX_DIAGNOSTIC_PATHS) {
131
+ const statusAgeMs = Date.now() - kb.updated_at;
132
+ const job = getIndexingJob(db, kb.id);
133
+ const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
134
+ const result = {
135
+ kb_id: kb.id,
136
+ kb_name: kb.name,
137
+ status: kb.status,
138
+ status_age_ms: statusAgeMs,
139
+ last_progress_age_ms: lastProgressAgeMs,
140
+ stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
141
+ stale_files: [],
142
+ orphan_files: [],
143
+ stale_file_count: 0,
144
+ orphan_file_count: 0,
145
+ stale_files_truncated: false,
146
+ orphan_files_truncated: false,
147
+ coverage_percent: 100,
148
+ total_source_files: 0,
149
+ indexed_files: kb.file_count,
150
+ skipped_files: createSkippedScanStats(),
151
+ job,
152
+ };
153
+ throwIfAborted(signal);
154
+ if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path))
155
+ return result;
156
+ const currentFiles = new Set();
157
+ let currentPathBytes = 0;
158
+ let isDirectory;
159
+ try {
160
+ isDirectory = (await statFile(kb.source_path)).isDirectory();
161
+ if (isDirectory) {
162
+ const skipped = createSkippedScanStats();
163
+ for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options), signal)) {
164
+ currentPathBytes += Buffer.byteLength(file.relPath);
165
+ if (currentFiles.size >= MAX_DIAGNOSTIC_WORKING_PATHS || currentPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
166
+ throw new Error("Diagnostic source path working-set limit exceeded");
167
+ }
168
+ currentFiles.add(file.relPath);
169
+ }
170
+ result.skipped_files = skipped;
171
+ }
172
+ else {
173
+ currentFiles.add(kb.source_path);
174
+ }
175
+ }
176
+ catch (error) {
177
+ throwIfAborted(signal);
178
+ result.coverage_percent = 0;
179
+ result.scan_error = diagnosticScanError(error);
180
+ return result;
181
+ }
182
+ result.total_source_files = currentFiles.size;
183
+ result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
184
+ const latestIndexedByFile = new Map();
185
+ let indexedPathBytes = 0;
186
+ let processed = 0;
187
+ for (const chunk of iterateChunksByKB(db, kb.id)) {
188
+ throwIfAborted(signal);
189
+ if (!latestIndexedByFile.has(chunk.file_path)) {
190
+ indexedPathBytes += Buffer.byteLength(chunk.file_path);
191
+ if (latestIndexedByFile.size >= MAX_DIAGNOSTIC_WORKING_PATHS ||
192
+ indexedPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
193
+ result.scan_error = "Diagnostic indexed path working-set limit exceeded";
194
+ result.orphan_files_truncated = true;
195
+ result.stale_files_truncated = true;
196
+ return result;
197
+ }
198
+ }
199
+ const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
200
+ if (chunk.indexed_at > currentLatest)
201
+ latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
202
+ processed++;
203
+ if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
204
+ await yieldToEventLoop();
205
+ }
206
+ let remainingDiagnosticPaths = maxDiagnosticPaths;
207
+ for (const filePath of latestIndexedByFile.keys()) {
208
+ throwIfAborted(signal);
209
+ if (!currentFiles.has(filePath)) {
210
+ result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
211
+ if (remainingDiagnosticPaths > 0) {
212
+ result.orphan_files.push(filePath);
213
+ remainingDiagnosticPaths--;
214
+ }
215
+ else
216
+ result.orphan_files_truncated = true;
217
+ }
218
+ }
219
+ processed = 0;
220
+ for (const relPath of currentFiles) {
221
+ throwIfAborted(signal);
222
+ const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
223
+ try {
224
+ const mtime = (await statFile(absPath)).mtimeMs;
225
+ const latestIndexed = latestIndexedByFile.get(relPath);
226
+ if (latestIndexed !== undefined && mtime > latestIndexed) {
227
+ result.stale_file_count = (result.stale_file_count ?? 0) + 1;
228
+ if (remainingDiagnosticPaths > 0) {
229
+ result.stale_files.push(relPath);
230
+ remainingDiagnosticPaths--;
231
+ }
232
+ else
233
+ result.stale_files_truncated = true;
104
234
  }
105
235
  }
106
236
  catch {
107
237
  /* file unreadable — skip */
108
238
  }
239
+ processed++;
240
+ if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
241
+ await yieldToEventLoop();
109
242
  }
110
243
  return result;
111
244
  }
@@ -1,5 +1,6 @@
1
1
  import { type DiagnosticResult } from "./diagnostics/health.ts";
2
2
  import type { EmbeddingCompatibility } from "./embedding/provider.ts";
3
+ import { type FilesystemAccessPolicy } from "./filesystem-access.ts";
3
4
  import { createSkippedScanStats } from "./indexer/chunker.ts";
4
5
  import { type RankingDiagnostics } from "./search/ranking.ts";
5
6
  import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
@@ -181,13 +182,14 @@ export declare class KnowledgeEngine {
181
182
  initialize(knowledgeDir: string): Promise<void>;
182
183
  private vectorPathFor;
183
184
  private deleteVectorFile;
184
- plan(source: string, options?: AddOptions, signal?: AbortSignal): IndexPlan;
185
- add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
185
+ plan(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): IndexPlan;
186
+ planAsync(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<IndexPlan>;
187
+ add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions, access?: FilesystemAccessPolicy): Promise<{
186
188
  kb: KnowledgeBase;
187
189
  chunkCount: number;
188
190
  }>;
189
191
  private addUnlocked;
190
- update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
192
+ update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
191
193
  added: number;
192
194
  removed: number;
193
195
  unchanged: number;
@@ -195,15 +197,18 @@ export declare class KnowledgeEngine {
195
197
  private runUpdate;
196
198
  search(query: string, options?: SearchOptions, signal?: AbortSignal): Promise<SearchResponse>;
197
199
  remove(nameOrId: string): boolean;
198
- list(signal?: AbortSignal): KnowledgeBase[];
200
+ list(signal?: AbortSignal, limit?: number): KnowledgeBase[];
199
201
  clear(): void;
200
202
  diagnose(signal?: AbortSignal): DiagnosticResult[];
201
- statusSnapshot(signal?: AbortSignal): KnowledgeStatusSnapshot;
203
+ diagnoseAsync(signal?: AbortSignal, knowledgeBasesOverride?: KnowledgeBase[], maxDiagnosticPaths?: number): Promise<DiagnosticResult[]>;
204
+ statusSnapshot(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): KnowledgeStatusSnapshot;
205
+ statusSnapshotAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<KnowledgeStatusSnapshot>;
202
206
  symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
203
- doctor(signal?: AbortSignal): DoctorReport;
204
- exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;
207
+ doctor(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): DoctorReport;
208
+ doctorAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<DoctorReport>;
209
+ exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback, access?: FilesystemAccessPolicy): Promise<number>;
205
210
  private exportKBUnlocked;
206
- importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
211
+ importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
207
212
  kb: KnowledgeBase;
208
213
  chunkCount: number;
209
214
  }>;