pi-knowledge 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +63 -24
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +300 -187
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +63 -9
  22. package/dist/src/engine.js +959 -170
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +10 -0
  26. package/dist/src/indexer/chunker.js +284 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/search/tuning.d.ts +2 -0
  47. package/dist/src/search/tuning.js +2 -0
  48. package/dist/src/secure-files.d.ts +17 -0
  49. package/dist/src/secure-files.js +184 -0
  50. package/dist/src/status.js +12 -7
  51. package/dist/src/storage/sqlite.d.ts +4 -1
  52. package/dist/src/storage/sqlite.js +46 -3
  53. package/dist/src/type-guards.d.ts +1 -0
  54. package/dist/src/type-guards.js +3 -0
  55. package/dist/src/watcher/file-watcher.d.ts +3 -1
  56. package/dist/src/watcher/file-watcher.js +166 -32
  57. package/dist/src/windows-security.d.ts +4 -0
  58. package/dist/src/windows-security.js +47 -0
  59. package/docs/configuration.md +42 -10
  60. package/docs/daemon-profiles.md +302 -0
  61. package/docs/mcp-clients.md +266 -0
  62. package/docs/offline-mode.md +6 -6
  63. package/package.json +28 -2
@@ -1,8 +1,14 @@
1
1
  import { existsSync, statSync } from "node:fs";
2
+ import { stat as statFile } from "node:fs/promises";
2
3
  import { join } from "node:path";
3
- import { createSkippedScanStats, iterateScannableFiles, } from "../indexer/chunker.js";
4
+ import { setImmediate as yieldToEventLoop } from "node:timers/promises";
5
+ import { createSkippedScanStats, iterateScannableFiles, iterateScannableFilesAsync, } from "../indexer/chunker.js";
4
6
  import { getIndexingJob, iterateChunksByKB } from "../storage/sqlite.js";
5
7
  const DEFAULT_STALE_INDEXING_MS = 10 * 60 * 1000;
8
+ const MAX_DIAGNOSTIC_PATHS = 1_000;
9
+ const DIAGNOSTIC_YIELD_ITEMS = 1_000;
10
+ const MAX_DIAGNOSTIC_WORKING_PATHS = 100_000;
11
+ const MAX_DIAGNOSTIC_WORKING_PATH_BYTES = 64 * 1024 * 1024;
6
12
  function staleIndexingMs() {
7
13
  const configured = Number(process.env.PI_KNOWLEDGE_STALE_INDEXING_MS);
8
14
  return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_STALE_INDEXING_MS;
@@ -30,6 +36,10 @@ function throwIfAborted(signal) {
30
36
  if (signal?.aborted)
31
37
  throw new Error("Cancelled");
32
38
  }
39
+ function diagnosticScanError(error) {
40
+ const message = error instanceof Error ? error.message : String(error);
41
+ return message.length > 512 ? `${message.slice(0, 512)}…` : message;
42
+ }
33
43
  export function diagnoseKB(db, kb, signal) {
34
44
  const statusAgeMs = Date.now() - kb.updated_at;
35
45
  const job = getIndexingJob(db, kb.id);
@@ -43,6 +53,10 @@ export function diagnoseKB(db, kb, signal) {
43
53
  stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
44
54
  stale_files: [],
45
55
  orphan_files: [],
56
+ stale_file_count: 0,
57
+ orphan_file_count: 0,
58
+ stale_files_truncated: false,
59
+ orphan_files_truncated: false,
46
60
  coverage_percent: 100,
47
61
  total_source_files: 0,
48
62
  indexed_files: kb.file_count,
@@ -69,9 +83,11 @@ export function diagnoseKB(db, kb, signal) {
69
83
  currentFiles.add(kb.source_path);
70
84
  }
71
85
  }
72
- catch {
86
+ catch (error) {
73
87
  if (signal?.aborted)
74
88
  throw new Error("Cancelled");
89
+ result.coverage_percent = 0;
90
+ result.scan_error = diagnosticScanError(error);
75
91
  return result;
76
92
  }
77
93
  result.total_source_files = currentFiles.size;
@@ -90,6 +106,7 @@ export function diagnoseKB(db, kb, signal) {
90
106
  throwIfAborted(signal);
91
107
  if (!currentFiles.has(filePath)) {
92
108
  result.orphan_files.push(filePath);
109
+ result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
93
110
  }
94
111
  }
95
112
  // Staleness detection: source files modified after last indexing
@@ -101,11 +118,127 @@ export function diagnoseKB(db, kb, signal) {
101
118
  const latestIndexed = latestIndexedByFile.get(relPath);
102
119
  if (latestIndexed !== undefined && mtime > latestIndexed) {
103
120
  result.stale_files.push(relPath);
121
+ result.stale_file_count = (result.stale_file_count ?? 0) + 1;
122
+ }
123
+ }
124
+ catch {
125
+ /* file unreadable — skip */
126
+ }
127
+ }
128
+ return result;
129
+ }
130
+ export async function diagnoseKBAsync(db, kb, signal, maxDiagnosticPaths = MAX_DIAGNOSTIC_PATHS) {
131
+ const statusAgeMs = Date.now() - kb.updated_at;
132
+ const job = getIndexingJob(db, kb.id);
133
+ const lastProgressAgeMs = job?.status === "running" ? Date.now() - job.last_progress_at : statusAgeMs;
134
+ const result = {
135
+ kb_id: kb.id,
136
+ kb_name: kb.name,
137
+ status: kb.status,
138
+ status_age_ms: statusAgeMs,
139
+ last_progress_age_ms: lastProgressAgeMs,
140
+ stuck_indexing: kb.status === "indexing" && lastProgressAgeMs > staleIndexingMs(),
141
+ stale_files: [],
142
+ orphan_files: [],
143
+ stale_file_count: 0,
144
+ orphan_file_count: 0,
145
+ stale_files_truncated: false,
146
+ orphan_files_truncated: false,
147
+ coverage_percent: 100,
148
+ total_source_files: 0,
149
+ indexed_files: kb.file_count,
150
+ skipped_files: createSkippedScanStats(),
151
+ job,
152
+ };
153
+ throwIfAborted(signal);
154
+ if (!kb.source_path || kb.source_type === "url" || !existsSync(kb.source_path))
155
+ return result;
156
+ const currentFiles = new Set();
157
+ let currentPathBytes = 0;
158
+ let isDirectory;
159
+ try {
160
+ isDirectory = (await statFile(kb.source_path)).isDirectory();
161
+ if (isDirectory) {
162
+ const skipped = createSkippedScanStats();
163
+ for await (const file of iterateScannableFilesAsync(kb.source_path, skipped, scanOptionsFromSourceOptions(kb.source_options), signal)) {
164
+ currentPathBytes += Buffer.byteLength(file.relPath);
165
+ if (currentFiles.size >= MAX_DIAGNOSTIC_WORKING_PATHS || currentPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
166
+ throw new Error("Diagnostic source path working-set limit exceeded");
167
+ }
168
+ currentFiles.add(file.relPath);
169
+ }
170
+ result.skipped_files = skipped;
171
+ }
172
+ else {
173
+ currentFiles.add(kb.source_path);
174
+ }
175
+ }
176
+ catch (error) {
177
+ throwIfAborted(signal);
178
+ result.coverage_percent = 0;
179
+ result.scan_error = diagnosticScanError(error);
180
+ return result;
181
+ }
182
+ result.total_source_files = currentFiles.size;
183
+ result.coverage_percent = currentFiles.size > 0 ? Math.round((result.indexed_files / currentFiles.size) * 100) : 100;
184
+ const latestIndexedByFile = new Map();
185
+ let indexedPathBytes = 0;
186
+ let processed = 0;
187
+ for (const chunk of iterateChunksByKB(db, kb.id)) {
188
+ throwIfAborted(signal);
189
+ if (!latestIndexedByFile.has(chunk.file_path)) {
190
+ indexedPathBytes += Buffer.byteLength(chunk.file_path);
191
+ if (latestIndexedByFile.size >= MAX_DIAGNOSTIC_WORKING_PATHS ||
192
+ indexedPathBytes > MAX_DIAGNOSTIC_WORKING_PATH_BYTES) {
193
+ result.scan_error = "Diagnostic indexed path working-set limit exceeded";
194
+ result.orphan_files_truncated = true;
195
+ result.stale_files_truncated = true;
196
+ return result;
197
+ }
198
+ }
199
+ const currentLatest = latestIndexedByFile.get(chunk.file_path) ?? 0;
200
+ if (chunk.indexed_at > currentLatest)
201
+ latestIndexedByFile.set(chunk.file_path, chunk.indexed_at);
202
+ processed++;
203
+ if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
204
+ await yieldToEventLoop();
205
+ }
206
+ let remainingDiagnosticPaths = maxDiagnosticPaths;
207
+ for (const filePath of latestIndexedByFile.keys()) {
208
+ throwIfAborted(signal);
209
+ if (!currentFiles.has(filePath)) {
210
+ result.orphan_file_count = (result.orphan_file_count ?? 0) + 1;
211
+ if (remainingDiagnosticPaths > 0) {
212
+ result.orphan_files.push(filePath);
213
+ remainingDiagnosticPaths--;
214
+ }
215
+ else
216
+ result.orphan_files_truncated = true;
217
+ }
218
+ }
219
+ processed = 0;
220
+ for (const relPath of currentFiles) {
221
+ throwIfAborted(signal);
222
+ const absPath = isDirectory ? join(kb.source_path, relPath) : relPath;
223
+ try {
224
+ const mtime = (await statFile(absPath)).mtimeMs;
225
+ const latestIndexed = latestIndexedByFile.get(relPath);
226
+ if (latestIndexed !== undefined && mtime > latestIndexed) {
227
+ result.stale_file_count = (result.stale_file_count ?? 0) + 1;
228
+ if (remainingDiagnosticPaths > 0) {
229
+ result.stale_files.push(relPath);
230
+ remainingDiagnosticPaths--;
231
+ }
232
+ else
233
+ result.stale_files_truncated = true;
104
234
  }
105
235
  }
106
236
  catch {
107
237
  /* file unreadable — skip */
108
238
  }
239
+ processed++;
240
+ if (processed % DIAGNOSTIC_YIELD_ITEMS === 0)
241
+ await yieldToEventLoop();
109
242
  }
110
243
  return result;
111
244
  }
@@ -1,5 +1,6 @@
1
1
  import { type DiagnosticResult } from "./diagnostics/health.ts";
2
2
  import type { EmbeddingCompatibility } from "./embedding/provider.ts";
3
+ import { type FilesystemAccessPolicy } from "./filesystem-access.ts";
3
4
  import { createSkippedScanStats } from "./indexer/chunker.ts";
4
5
  import { type RankingDiagnostics } from "./search/ranking.ts";
5
6
  import { type SearchProfile, type SearchTuningSummary } from "./search/tuning.ts";
@@ -38,6 +39,40 @@ export interface SearchResult {
38
39
  };
39
40
  }
40
41
  export { CURRENT_EMBEDDING_MODEL } from "./embedding/provider.ts";
42
+ export type SearchMissReason = "no_knowledge_bases" | "knowledge_base_not_found" | "no_searchable_knowledge_bases" | "no_indexed_chunks" | "embedding_incompatible" | "no_retrieval_candidates" | "threshold_filtered" | "evidence_filtered" | "metadata_filtered" | "reranker_empty" | "page_out_of_range" | "no_match";
43
+ export type SearchMissAction = "add_knowledge_base" | "fix_kb_id" | "wait_for_indexing" | "fix_knowledge_base" | "fix_embedding" | "try_semantic" | "try_semantic_hybrid" | "relax_filter" | "reset_offset" | "rephrase";
44
+ export interface SearchKnowledgeBaseSummary {
45
+ id: string;
46
+ name: string;
47
+ status: KnowledgeBase["status"];
48
+ chunk_count: number;
49
+ }
50
+ export interface SearchAttemptSummary {
51
+ mode: SearchMode;
52
+ reason?: SearchMissReason;
53
+ retrieved_candidates: number;
54
+ threshold_candidates: number;
55
+ eligible_candidates: number;
56
+ filtered_candidates: number;
57
+ result_count: number;
58
+ }
59
+ export interface SearchMiss {
60
+ reason: SearchMissReason;
61
+ action: SearchMissAction;
62
+ requested_kb_id?: string;
63
+ file_type?: string;
64
+ path_pattern?: string;
65
+ min_hybrid_score?: number;
66
+ searched_kbs: SearchKnowledgeBaseSummary[];
67
+ skipped_kbs: SearchKnowledgeBaseSummary[];
68
+ retrieved_candidates: number;
69
+ threshold_candidates: number;
70
+ eligible_candidates: number;
71
+ filtered_candidates: number;
72
+ ranked_candidates: number;
73
+ total_results: number;
74
+ attempts?: SearchAttemptSummary[];
75
+ }
41
76
  export interface SearchResponse {
42
77
  results: SearchResult[];
43
78
  total_count: number;
@@ -45,8 +80,8 @@ export interface SearchResponse {
45
80
  warnings?: string[];
46
81
  mode_used?: SearchMode;
47
82
  retry_modes?: SearchMode[];
48
- suggestions?: string[];
49
83
  tuning?: SearchTuningSummary;
84
+ miss?: SearchMiss;
50
85
  }
51
86
  export type ProgressCallback = (msg: string) => void;
52
87
  export interface AddOptions {
@@ -96,6 +131,20 @@ export interface SymbolSearchOptions {
96
131
  offset?: number;
97
132
  exact?: boolean;
98
133
  }
134
+ export type SymbolSearchMissReason = "no_knowledge_bases" | "knowledge_base_not_found" | "no_searchable_knowledge_bases" | "symbol_index_empty" | "exact_name_not_found" | "metadata_filtered" | "page_out_of_range" | "no_match";
135
+ export type SymbolSearchMissAction = "add_knowledge_base" | "fix_kb_id" | "wait_for_indexing" | "fix_knowledge_base" | "rebuild_symbol_index" | "broaden_symbol_search" | "relax_filter" | "reset_offset" | "use_text_search";
136
+ export interface SymbolSearchMiss {
137
+ reason: SymbolSearchMissReason;
138
+ action: SymbolSearchMissAction;
139
+ requested_kb_id?: string;
140
+ kind?: KnowledgeSymbol["kind"];
141
+ file_pattern?: string;
142
+ exact: boolean;
143
+ searched_kbs: SearchKnowledgeBaseSummary[];
144
+ skipped_kbs: SearchKnowledgeBaseSummary[];
145
+ indexed_symbols: number;
146
+ unfiltered_matches: number;
147
+ }
99
148
  export interface SymbolSearchResponse {
100
149
  results: Array<{
101
150
  name: string;
@@ -112,6 +161,7 @@ export interface SymbolSearchResponse {
112
161
  }>;
113
162
  total_count: number;
114
163
  has_more: boolean;
164
+ miss?: SymbolSearchMiss;
115
165
  }
116
166
  export declare const CURRENT_URL_STRUCTURE_POLICY = 1;
117
167
  export declare const CURRENT_EMBEDDING_INPUT_POLICY = 2;
@@ -132,13 +182,14 @@ export declare class KnowledgeEngine {
132
182
  initialize(knowledgeDir: string): Promise<void>;
133
183
  private vectorPathFor;
134
184
  private deleteVectorFile;
135
- plan(source: string, options?: AddOptions, signal?: AbortSignal): IndexPlan;
136
- add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions): Promise<{
185
+ plan(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): IndexPlan;
186
+ planAsync(source: string, options?: AddOptions, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<IndexPlan>;
187
+ add(source: string, name: string, onProgress?: ProgressCallback, signal?: AbortSignal, options?: AddOptions, access?: FilesystemAccessPolicy): Promise<{
137
188
  kb: KnowledgeBase;
138
189
  chunkCount: number;
139
190
  }>;
140
191
  private addUnlocked;
141
- update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
192
+ update(nameOrId: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
142
193
  added: number;
143
194
  removed: number;
144
195
  unchanged: number;
@@ -146,15 +197,18 @@ export declare class KnowledgeEngine {
146
197
  private runUpdate;
147
198
  search(query: string, options?: SearchOptions, signal?: AbortSignal): Promise<SearchResponse>;
148
199
  remove(nameOrId: string): boolean;
149
- list(signal?: AbortSignal): KnowledgeBase[];
200
+ list(signal?: AbortSignal, limit?: number): KnowledgeBase[];
150
201
  clear(): void;
151
202
  diagnose(signal?: AbortSignal): DiagnosticResult[];
152
- statusSnapshot(signal?: AbortSignal): KnowledgeStatusSnapshot;
203
+ diagnoseAsync(signal?: AbortSignal, knowledgeBasesOverride?: KnowledgeBase[], maxDiagnosticPaths?: number): Promise<DiagnosticResult[]>;
204
+ statusSnapshot(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): KnowledgeStatusSnapshot;
205
+ statusSnapshotAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<KnowledgeStatusSnapshot>;
153
206
  symbolSearch(query: string, options?: SymbolSearchOptions, signal?: AbortSignal): SymbolSearchResponse;
154
- doctor(signal?: AbortSignal): DoctorReport;
155
- exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback): Promise<number>;
207
+ doctor(signal?: AbortSignal, diagnosticsOverride?: DiagnosticResult[], knowledgeBasesOverride?: KnowledgeBase[]): DoctorReport;
208
+ doctorAsync(signal?: AbortSignal, maxKnowledgeBases?: number, maxDiagnosticPaths?: number): Promise<DoctorReport>;
209
+ exportKB(nameOrId: string, outputPath: string, signal?: AbortSignal, onProgress?: ProgressCallback, access?: FilesystemAccessPolicy): Promise<number>;
156
210
  private exportKBUnlocked;
157
- importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal): Promise<{
211
+ importKB(inputPath: string, onProgress?: ProgressCallback, signal?: AbortSignal, access?: FilesystemAccessPolicy): Promise<{
158
212
  kb: KnowledgeBase;
159
213
  chunkCount: number;
160
214
  }>;