pi-knowledge 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +59 -22
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +145 -166
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +13 -8
  22. package/dist/src/engine.js +569 -100
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +10 -0
  26. package/dist/src/indexer/chunker.js +284 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/secure-files.d.ts +17 -0
  47. package/dist/src/secure-files.js +184 -0
  48. package/dist/src/status.js +12 -7
  49. package/dist/src/storage/sqlite.d.ts +4 -1
  50. package/dist/src/storage/sqlite.js +46 -3
  51. package/dist/src/type-guards.d.ts +1 -0
  52. package/dist/src/type-guards.js +3 -0
  53. package/dist/src/watcher/file-watcher.d.ts +3 -1
  54. package/dist/src/watcher/file-watcher.js +166 -32
  55. package/dist/src/windows-security.d.ts +4 -0
  56. package/dist/src/windows-security.js +47 -0
  57. package/docs/configuration.md +39 -9
  58. package/docs/daemon-profiles.md +302 -0
  59. package/docs/mcp-clients.md +266 -0
  60. package/docs/offline-mode.md +6 -6
  61. package/package.json +28 -2
@@ -0,0 +1,184 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
3
+ import { basename, dirname, join, resolve } from "node:path";
4
+ import { assertWindowsPathAncestorsTrusted, assertWindowsPathPrivate, hardenWindowsPath } from "./windows-security.js";
5
+ export function assertOwnedDirectory(path) {
6
+ const info = lstatSync(path);
7
+ if (info.isSymbolicLink() || !info.isDirectory()) {
8
+ throw new Error(`Expected a real directory owned by the current user: ${path}`);
9
+ }
10
+ if (process.platform === "win32") {
11
+ assertWindowsPathPrivate(path);
12
+ assertWindowsPathAncestorsTrusted(path);
13
+ }
14
+ else {
15
+ if (typeof process.getuid === "function" && info.uid !== process.getuid()) {
16
+ throw new Error(`Directory is not owned by the current user: ${path}`);
17
+ }
18
+ if ((info.mode & 0o022) !== 0)
19
+ throw new Error(`Directory must not be group/world writable: ${path}`);
20
+ }
21
+ if (process.platform !== "win32" && typeof process.getuid === "function") {
22
+ const uid = process.getuid();
23
+ let current = realpathSync.native(path);
24
+ while (true) {
25
+ const ancestor = lstatSync(current);
26
+ if (ancestor.isSymbolicLink() || !ancestor.isDirectory()) {
27
+ throw new Error(`Path ancestor is not a real directory: ${current}`);
28
+ }
29
+ if (ancestor.uid !== uid && ancestor.uid !== 0) {
30
+ throw new Error(`Path ancestor is owned by an untrusted user: ${current}`);
31
+ }
32
+ if ((ancestor.mode & 0o022) !== 0 && (ancestor.mode & 0o1000) === 0) {
33
+ throw new Error(`Path ancestor permits untrusted rename operations: ${current}`);
34
+ }
35
+ const parent = dirname(current);
36
+ if (parent === current)
37
+ break;
38
+ current = parent;
39
+ }
40
+ }
41
+ }
42
+ export function ensurePrivateDirectory(path) {
43
+ mkdirSync(path, { recursive: true, mode: 0o700 });
44
+ hardenWindowsPath(path, true);
45
+ assertOwnedDirectory(path);
46
+ return realpathSync.native(path);
47
+ }
48
+ export function ensureOwnerOnlyDirectory(path) {
49
+ const canonical = ensurePrivateDirectory(path);
50
+ if (process.platform !== "win32") {
51
+ chmodSync(canonical, 0o700);
52
+ const info = statSync(canonical);
53
+ if ((info.mode & 0o077) !== 0)
54
+ throw new Error(`Directory must use owner-only permissions: ${canonical}`);
55
+ }
56
+ return canonical;
57
+ }
58
+ export function fsyncDirectory(path) {
59
+ try {
60
+ const fd = openSync(path, "r");
61
+ try {
62
+ fsyncSync(fd);
63
+ }
64
+ finally {
65
+ closeSync(fd);
66
+ }
67
+ }
68
+ catch {
69
+ // Directory fsync is unavailable on some supported platforms.
70
+ }
71
+ }
72
+ export function prepareSafeNewFile(path) {
73
+ const requested = resolve(path);
74
+ const parentPath = realpathSync.native(dirname(requested));
75
+ assertOwnedDirectory(parentPath);
76
+ const outputPath = join(parentPath, basename(requested));
77
+ if (existsSync(outputPath))
78
+ throw new Error(`Refusing to overwrite existing file: ${outputPath}`);
79
+ const parent = statSync(parentPath);
80
+ return {
81
+ outputPath,
82
+ parentPath,
83
+ parentDevice: parent.dev,
84
+ parentInode: parent.ino,
85
+ tempPath: join(parentPath, `.${basename(requested)}.${randomUUID()}.tmp`),
86
+ };
87
+ }
88
+ export function publishNewFile(tempPath, target) {
89
+ const currentParent = statSync(target.parentPath);
90
+ if (currentParent.dev !== target.parentDevice || currentParent.ino !== target.parentInode) {
91
+ throw new Error(`Export directory changed during publication: ${target.parentPath}`);
92
+ }
93
+ linkSync(tempPath, target.outputPath);
94
+ fsyncDirectory(target.parentPath);
95
+ rmSync(tempPath, { force: true });
96
+ }
97
+ export function hardenPrivateFile(path, label) {
98
+ const info = lstatSync(path);
99
+ if (info.isSymbolicLink() || !info.isFile())
100
+ throw new Error(`Invalid ${label}: ${path}`);
101
+ if (process.platform === "win32") {
102
+ hardenWindowsPath(path, false);
103
+ }
104
+ else {
105
+ if (typeof process.getuid === "function" && info.uid !== process.getuid()) {
106
+ throw new Error(`${label} is not owned by the current user: ${path}`);
107
+ }
108
+ chmodSync(path, 0o600);
109
+ }
110
+ }
111
+ export function readPrivateJson(path, label) {
112
+ assertOwnedDirectory(dirname(path));
113
+ const direct = lstatSync(path);
114
+ if (direct.isSymbolicLink() || !direct.isFile())
115
+ throw new Error(`Invalid ${label}: ${path}`);
116
+ if (process.platform === "win32") {
117
+ assertWindowsPathPrivate(path);
118
+ }
119
+ else {
120
+ if (typeof process.getuid === "function" && direct.uid !== process.getuid()) {
121
+ throw new Error(`${label} is not owned by the current user: ${path}`);
122
+ }
123
+ if ((direct.mode & 0o077) !== 0)
124
+ throw new Error(`${label} permissions are too broad: ${path}`);
125
+ }
126
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
127
+ try {
128
+ const opened = fstatSync(fd);
129
+ if (opened.dev !== direct.dev || opened.ino !== direct.ino)
130
+ throw new Error(`${label} changed while opening: ${path}`);
131
+ return JSON.parse(readFileSync(fd, "utf8"));
132
+ }
133
+ finally {
134
+ closeSync(fd);
135
+ }
136
+ }
137
+ export function writeJsonAtomic(path, value) {
138
+ const parent = ensurePrivateDirectory(dirname(path));
139
+ const temp = join(parent, `.${randomUUID()}.tmp`);
140
+ let fd;
141
+ try {
142
+ fd = openSync(temp, "wx", 0o600);
143
+ writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, "utf8");
144
+ fsyncSync(fd);
145
+ closeSync(fd);
146
+ fd = undefined;
147
+ renameSync(temp, path);
148
+ hardenWindowsPath(path, false);
149
+ fsyncDirectory(parent);
150
+ }
151
+ finally {
152
+ if (fd !== undefined)
153
+ closeSync(fd);
154
+ rmSync(temp, { force: true });
155
+ }
156
+ }
157
+ export function writeJsonExclusive(path, value) {
158
+ const parent = ensurePrivateDirectory(dirname(path));
159
+ const temp = join(parent, `.${randomUUID()}.tmp`);
160
+ let fd;
161
+ try {
162
+ fd = openSync(temp, "wx", 0o600);
163
+ writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, "utf8");
164
+ fsyncSync(fd);
165
+ closeSync(fd);
166
+ fd = undefined;
167
+ try {
168
+ linkSync(temp, path);
169
+ }
170
+ catch (error) {
171
+ if (error.code === "EEXIST")
172
+ return false;
173
+ throw error;
174
+ }
175
+ hardenPrivateFile(path, "knowledge profile registry");
176
+ fsyncDirectory(parent);
177
+ return true;
178
+ }
179
+ finally {
180
+ if (fd !== undefined)
181
+ closeSync(fd);
182
+ rmSync(temp, { force: true });
183
+ }
184
+ }
@@ -48,11 +48,16 @@ export function formatKnowledgeStatus(options) {
48
48
  if (diagnostic.job.error_message)
49
49
  lines.push(` error: ${diagnostic.job.error_message}`);
50
50
  }
51
- lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
52
- if (diagnostic.stale_files.length > 0)
53
- lines.push(` ⚠️ stale: ${diagnostic.stale_files.length} files modified since last index`);
54
- if (diagnostic.orphan_files.length > 0)
55
- lines.push(` ⚠️ orphans: ${diagnostic.orphan_files.length} chunks reference deleted files`);
51
+ if (diagnostic.scan_error)
52
+ lines.push(` ⚠️ source scan failed: ${diagnostic.scan_error}`);
53
+ else
54
+ lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
55
+ const staleCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
56
+ const orphanCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
57
+ if (staleCount > 0)
58
+ lines.push(` ⚠️ stale: ${staleCount} files modified since last index`);
59
+ if (orphanCount > 0)
60
+ lines.push(` ⚠️ orphans: ${orphanCount} chunks reference deleted files`);
56
61
  if (diagnostic.stuck_indexing)
57
62
  lines.push(` ⚠️ indexing appears stuck for ${Math.round(diagnostic.last_progress_age_ms / 60_000)}m; remove and rebuild if no pi process is actively indexing it`);
58
63
  if (diagnostic.skipped_files.total > 0)
@@ -61,8 +66,8 @@ export function formatKnowledgeStatus(options) {
61
66
  .map(([reason, count]) => `${reason}: ${count}`)
62
67
  .join(", ")})`);
63
68
  }
64
- const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.stale_files.length, 0);
65
- const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.orphan_files.length, 0);
69
+ const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + (diagnostic.stale_file_count ?? diagnostic.stale_files.length), 0);
70
+ const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + (diagnostic.orphan_file_count ?? diagnostic.orphan_files.length), 0);
66
71
  const totalStuck = snapshot.diagnostics.filter((diagnostic) => diagnostic.stuck_indexing).length;
67
72
  const totalIndexing = snapshot.knowledge_bases.filter((kb) => kb.status === "indexing" && !diagnosticsByKb.get(kb.id)?.stuck_indexing).length;
68
73
  const totalErrors = snapshot.knowledge_bases.filter((kb) => kb.status === "error").length;
@@ -8,6 +8,7 @@ export interface KnowledgeBase {
8
8
  source_options: string | null;
9
9
  created_at: number;
10
10
  updated_at: number;
11
+ indexed_at: number;
11
12
  chunk_count: number;
12
13
  file_count: number;
13
14
  embedding_model: string;
@@ -83,10 +84,11 @@ export declare function createKB(db: Database.Database, opts: {
83
84
  }): KnowledgeBase;
84
85
  export declare function getKB(db: Database.Database, id: string): KnowledgeBase | undefined;
85
86
  export declare function getKBByName(db: Database.Database, name: string): KnowledgeBase | undefined;
86
- export declare function listKBs(db: Database.Database): KnowledgeBase[];
87
+ export declare function listKBs(db: Database.Database, limit?: number): KnowledgeBase[];
87
88
  export declare function deleteKB(db: Database.Database, id: string): void;
88
89
  export declare function updateKBStatus(db: Database.Database, id: string, status: KnowledgeBase["status"]): void;
89
90
  export declare function updateKBCounts(db: Database.Database, id: string, chunkCount: number, fileCount: number): void;
91
+ export declare function markKBIndexed(db: Database.Database, id: string): void;
90
92
  export declare function updateKBSourceOptions(db: Database.Database, id: string, sourceOptions: string): void;
91
93
  export declare function updateKBEmbeddingMetadata(db: Database.Database, id: string, embeddingModel: string, embeddingSignature: string | null, embeddingDimension: number | null): void;
92
94
  export declare function startIndexingJob(db: Database.Database, kbId: string, operation: IndexingJob["operation"], message: string): void;
@@ -123,6 +125,7 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
123
125
  export declare function getChunkCount(db: Database.Database, kbId: string): number;
124
126
  export declare function getFileCount(db: Database.Database, kbId: string): number;
125
127
  export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
128
+ export declare function getSymbolsByFile(db: Database.Database, kbId: string, filePath: string): KnowledgeSymbolInsert[];
126
129
  export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
127
130
  export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
128
131
  export declare function searchSymbols(db: Database.Database, query: string, options?: {
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
4
4
  import { homedir } from "node:os";
5
5
  import { dirname, join } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
- const SCHEMA_VERSION = 5;
7
+ const SCHEMA_VERSION = 6;
8
8
  const ITERATION_BATCH_SIZE = 500;
9
9
  const SCHEMA_SQL = `
10
10
  CREATE TABLE IF NOT EXISTS schema_version (
@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
20
20
  source_options TEXT,
21
21
  created_at INTEGER NOT NULL,
22
22
  updated_at INTEGER NOT NULL,
23
+ indexed_at INTEGER NOT NULL DEFAULT 0,
23
24
  chunk_count INTEGER NOT NULL DEFAULT 0,
24
25
  file_count INTEGER NOT NULL DEFAULT 0,
25
26
  embedding_model TEXT NOT NULL DEFAULT 'multilingual-e5-small',
@@ -316,6 +317,30 @@ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
316
317
  }
317
318
  continue;
318
319
  }
320
+ if (v === 6) {
321
+ const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all();
322
+ if (!columns.some((column) => column.name === "indexed_at")) {
323
+ db.exec("ALTER TABLE knowledge_bases ADD COLUMN indexed_at INTEGER NOT NULL DEFAULT 0");
324
+ db.exec(`
325
+ UPDATE knowledge_bases
326
+ SET indexed_at = COALESCE(
327
+ (
328
+ SELECT updated_at
329
+ FROM indexing_jobs
330
+ WHERE indexing_jobs.kb_id = knowledge_bases.id
331
+ AND indexing_jobs.status = 'succeeded'
332
+ ),
333
+ (
334
+ SELECT MAX(indexed_at)
335
+ FROM chunks
336
+ WHERE chunks.kb_id = knowledge_bases.id
337
+ ),
338
+ 0
339
+ )
340
+ `);
341
+ }
342
+ continue;
343
+ }
319
344
  if (migrations[v])
320
345
  db.exec(migrations[v]);
321
346
  }
@@ -340,8 +365,13 @@ export function getKB(db, id) {
340
365
  export function getKBByName(db, name) {
341
366
  return db.prepare("SELECT * FROM knowledge_bases WHERE name = ?").get(name);
342
367
  }
343
- export function listKBs(db) {
344
- return db.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC").all();
368
+ export function listKBs(db, limit) {
369
+ if (limit === undefined) {
370
+ return db.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC").all();
371
+ }
372
+ return db
373
+ .prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC LIMIT ?")
374
+ .all(Math.max(0, Math.trunc(limit)));
345
375
  }
346
376
  export function deleteKB(db, id) {
347
377
  db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
@@ -355,6 +385,10 @@ export function updateKBStatus(db, id, status) {
355
385
  export function updateKBCounts(db, id, chunkCount, fileCount) {
356
386
  db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(chunkCount, fileCount, Date.now(), id);
357
387
  }
388
+ export function markKBIndexed(db, id) {
389
+ const now = Date.now();
390
+ db.prepare("UPDATE knowledge_bases SET indexed_at = ?, updated_at = ? WHERE id = ?").run(now, now, id);
391
+ }
358
392
  export function updateKBSourceOptions(db, id, sourceOptions) {
359
393
  db.prepare("UPDATE knowledge_bases SET source_options = ?, updated_at = ? WHERE id = ?").run(sourceOptions, Date.now(), id);
360
394
  }
@@ -562,6 +596,15 @@ export function insertSymbols(db, kbId, symbols) {
562
596
  });
563
597
  insertMany(symbols);
564
598
  }
599
+ export function getSymbolsByFile(db, kbId, filePath) {
600
+ return db
601
+ .prepare(`SELECT name, kind, file_path, file_type, start_line, end_line,
602
+ container_name, signature, text, metadata_json
603
+ FROM symbols
604
+ WHERE kb_id = ? AND file_path = ?
605
+ ORDER BY indexed_at, rowid`)
606
+ .all(kbId, filePath);
607
+ }
565
608
  export function deleteSymbolsByKB(db, kbId) {
566
609
  db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
567
610
  }
@@ -0,0 +1 @@
1
+ export declare function isObjectRecord(value: unknown): value is Record<string, unknown>;
@@ -0,0 +1,3 @@
1
+ export function isObjectRecord(value) {
2
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3
+ }
@@ -1,5 +1,7 @@
1
1
  import { type ScanOptions } from "../indexer/chunker.ts";
2
- export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
2
+ export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): Promise<void>;
3
3
  export declare function stopWatcher(kbId: string): void;
4
4
  export declare function stopAllWatchers(): void;
5
+ export declare function stopAllWatchersAndWait(): Promise<void>;
6
+ export declare function getActiveWatcherScanCount(): number;
5
7
  export declare function getActiveWatcherCount(): number;
@@ -1,11 +1,24 @@
1
1
  import { existsSync, statSync, watch } from "node:fs";
2
- import { createSkippedScanStats, iterateScannableFiles } from "../indexer/chunker.js";
2
+ import { resolve } from "node:path";
3
+ import { setImmediate as yieldToEventLoop } from "node:timers/promises";
4
+ import { createSkippedScanStats, iterateScannableFileVisitsAsync } from "../indexer/chunker.js";
3
5
  const watchers = new Map();
4
6
  const pollers = new Map();
5
7
  const snapshots = new Map();
6
8
  const debounceTimers = new Map();
9
+ const scanRuns = new Map();
10
+ const scanControllers = new Map();
11
+ const scanGenerations = new Map();
12
+ const scanRequested = new Set();
13
+ const eventPaths = new Map();
14
+ const baselineEvents = new Set();
15
+ const baselineRootStamps = new Map();
16
+ let scanPermit = Promise.resolve();
17
+ const generations = new Map();
7
18
  const DEBOUNCE_MS = 2000;
8
19
  const POLL_MS = 2000;
20
+ const SCAN_YIELD_ENTRIES = 50;
21
+ const MAX_EVENT_PATHS = 1_024;
9
22
  function scheduleUpdate(kbId, onUpdate) {
10
23
  const existing = debounceTimers.get(kbId);
11
24
  if (existing)
@@ -15,56 +28,163 @@ function scheduleUpdate(kbId, onUpdate) {
15
28
  onUpdate(kbId);
16
29
  }, DEBOUNCE_MS));
17
30
  }
18
- function scanSnapshot(dirPath, options = {}) {
31
+ function rootStamp(path) {
32
+ try {
33
+ const info = statSync(path);
34
+ return `${info.mtimeMs}:${info.ctimeMs}:${info.size}`;
35
+ }
36
+ catch {
37
+ return undefined;
38
+ }
39
+ }
40
+ async function scanSnapshot(dirPath, options = {}, signal) {
19
41
  const snapshot = new Map();
20
42
  if (!existsSync(dirPath))
21
43
  return snapshot;
44
+ if (signal?.aborted)
45
+ return snapshot;
22
46
  const skipped = createSkippedScanStats();
23
- for (const file of iterateScannableFiles(dirPath, skipped, options)) {
24
- try {
25
- const stat = statSync(file.path);
26
- snapshot.set(file.path, `${stat.mtimeMs}:${stat.size}`);
47
+ let processed = 0;
48
+ for await (const file of iterateScannableFileVisitsAsync(dirPath, skipped, options, signal)) {
49
+ if (signal?.aborted)
50
+ return snapshot;
51
+ if (file) {
52
+ snapshot.set(file.path, `${file.mtimeMs}:${file.ctimeMs}:${file.size}:${file.inode}`);
27
53
  }
28
- catch {
29
- /* file disappeared or is unreadable */
54
+ processed++;
55
+ if (processed % SCAN_YIELD_ENTRIES === 0) {
56
+ await yieldToEventLoop();
57
+ if (signal?.aborted)
58
+ return snapshot;
30
59
  }
31
60
  }
32
61
  return snapshot;
33
62
  }
34
- function snapshotsDiffer(a, b) {
35
- if (a.size !== b.size)
63
+ async function withScanPermit(operation) {
64
+ const previous = scanPermit;
65
+ let release = () => { };
66
+ scanPermit = new Promise((resolve) => {
67
+ release = resolve;
68
+ });
69
+ await previous.catch(() => { });
70
+ try {
71
+ return await operation();
72
+ }
73
+ finally {
74
+ release();
75
+ }
76
+ }
77
+ function snapshotsDiffer(left, right) {
78
+ if (left.size !== right.size)
79
+ return true;
80
+ for (const [path, value] of left) {
81
+ if (right.get(path) !== value)
82
+ return true;
83
+ }
84
+ return false;
85
+ }
86
+ function eventTouchesSnapshot(path, snapshot) {
87
+ if (path === undefined)
36
88
  return true;
37
- for (const [path, value] of a) {
38
- if (b.get(path) !== value)
89
+ if (!snapshot)
90
+ return false;
91
+ if (snapshot.has(path))
92
+ return true;
93
+ const prefix = `${path}/`;
94
+ for (const candidate of snapshot.keys()) {
95
+ if (candidate.startsWith(prefix))
39
96
  return true;
40
97
  }
41
98
  return false;
42
99
  }
43
- function startPoller(kbId, dirPath, onUpdate, options = {}) {
44
- snapshots.set(kbId, scanSnapshot(dirPath, options));
45
- pollers.set(kbId, setInterval(() => {
46
- const previous = snapshots.get(kbId) ?? new Map();
47
- const next = scanSnapshot(dirPath, options);
48
- if (snapshotsDiffer(previous, next)) {
100
+ function requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive = true) {
101
+ if (generations.get(kbId) !== generation)
102
+ return Promise.resolve();
103
+ const existing = scanRuns.get(kbId);
104
+ if (existing) {
105
+ if (scanGenerations.get(kbId) === generation) {
106
+ if (followUpIfActive)
107
+ scanRequested.add(kbId);
108
+ return existing;
109
+ }
110
+ return existing.then(() => requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive), () => requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive));
111
+ }
112
+ const controller = new AbortController();
113
+ const run = (async () => {
114
+ do {
115
+ if (generations.get(kbId) !== generation)
116
+ return;
117
+ scanRequested.delete(kbId);
118
+ const next = await withScanPermit(() => scanSnapshot(dirPath, options, controller.signal));
119
+ if (controller.signal.aborted)
120
+ return;
121
+ if (generations.get(kbId) !== generation)
122
+ return;
123
+ const previous = snapshots.get(kbId);
49
124
  snapshots.set(kbId, next);
50
- scheduleUpdate(kbId, onUpdate);
125
+ const baselineStamp = baselineRootStamps.get(kbId);
126
+ baselineRootStamps.delete(kbId);
127
+ const baselineChanged = previous === undefined && baselineStamp !== undefined && rootStamp(dirPath) !== baselineStamp;
128
+ const events = eventPaths.get(kbId);
129
+ eventPaths.delete(kbId);
130
+ const eventRelevant = events !== undefined &&
131
+ [...events].some((path) => eventTouchesSnapshot(path, previous) || eventTouchesSnapshot(path, next));
132
+ if (baselineEvents.delete(kbId) ||
133
+ baselineChanged ||
134
+ (previous && snapshotsDiffer(previous, next)) ||
135
+ eventRelevant) {
136
+ scheduleUpdate(kbId, onUpdate);
137
+ }
138
+ } while (scanRequested.has(kbId));
139
+ })().finally(() => {
140
+ if (scanRuns.get(kbId) === run) {
141
+ scanRuns.delete(kbId);
142
+ scanControllers.delete(kbId);
143
+ scanGenerations.delete(kbId);
51
144
  }
52
- }, POLL_MS));
145
+ });
146
+ scanRuns.set(kbId, run);
147
+ scanControllers.set(kbId, controller);
148
+ scanGenerations.set(kbId, generation);
149
+ void run.catch(() => {
150
+ /* transient scan errors are retried by the polling fallback */
151
+ });
152
+ return run;
153
+ }
154
+ function startPoller(kbId, generation, dirPath, onUpdate, options = {}) {
155
+ const initialScan = requestScan(kbId, generation, dirPath, onUpdate, options, false);
156
+ pollers.set(kbId, setInterval(() => requestScan(kbId, generation, dirPath, onUpdate, options, false), POLL_MS));
157
+ return initialScan;
53
158
  }
54
159
  export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
55
160
  stopWatcher(kbId);
56
- startPoller(kbId, dirPath, onUpdate, options);
161
+ baselineRootStamps.set(kbId, rootStamp(dirPath));
162
+ const generation = {};
163
+ generations.set(kbId, generation);
164
+ const initialScan = startPoller(kbId, generation, dirPath, onUpdate, options);
57
165
  try {
58
- const watcher = watch(dirPath, { recursive: true }, () => {
59
- const previous = snapshots.get(kbId) ?? new Map();
60
- const next = scanSnapshot(dirPath, options);
61
- if (!snapshotsDiffer(previous, next))
166
+ const watcher = watch(dirPath, { recursive: true }, (_event, filename) => {
167
+ if (generations.get(kbId) !== generation)
62
168
  return;
63
- snapshots.set(kbId, next);
64
- scheduleUpdate(kbId, onUpdate);
169
+ if (!snapshots.has(kbId))
170
+ baselineEvents.add(kbId);
171
+ const paths = eventPaths.get(kbId) ?? new Set();
172
+ if (!paths.has(undefined)) {
173
+ if (filename === null || paths.size >= MAX_EVENT_PATHS) {
174
+ paths.clear();
175
+ paths.add(undefined);
176
+ }
177
+ else {
178
+ paths.add(resolve(dirPath, filename.toString()));
179
+ }
180
+ }
181
+ eventPaths.set(kbId, paths);
182
+ void requestScan(kbId, generation, dirPath, onUpdate, options);
65
183
  });
66
184
  watcher.on("error", () => {
67
- watchers.get(kbId)?.close();
185
+ if (watchers.get(kbId) !== watcher || generations.get(kbId) !== generation)
186
+ return;
187
+ watcher.close();
68
188
  watchers.delete(kbId);
69
189
  });
70
190
  watchers.set(kbId, watcher);
@@ -72,8 +192,11 @@ export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
72
192
  catch {
73
193
  /* polling fallback remains active */
74
194
  }
195
+ return initialScan;
75
196
  }
76
197
  export function stopWatcher(kbId) {
198
+ generations.delete(kbId);
199
+ scanControllers.get(kbId)?.abort();
77
200
  watchers.get(kbId)?.close();
78
201
  watchers.delete(kbId);
79
202
  const poller = pollers.get(kbId);
@@ -81,18 +204,29 @@ export function stopWatcher(kbId) {
81
204
  clearInterval(poller);
82
205
  pollers.delete(kbId);
83
206
  }
207
+ scanRequested.delete(kbId);
84
208
  snapshots.delete(kbId);
85
- const t = debounceTimers.get(kbId);
86
- if (t) {
87
- clearTimeout(t);
209
+ const timer = debounceTimers.get(kbId);
210
+ if (timer) {
211
+ clearTimeout(timer);
88
212
  debounceTimers.delete(kbId);
89
213
  }
214
+ eventPaths.delete(kbId);
215
+ baselineRootStamps.delete(kbId);
216
+ baselineEvents.delete(kbId);
90
217
  }
91
218
  export function stopAllWatchers() {
92
- const ids = new Set([...watchers.keys(), ...pollers.keys()]);
219
+ const ids = new Set([...watchers.keys(), ...pollers.keys(), ...scanRuns.keys()]);
93
220
  for (const id of ids)
94
221
  stopWatcher(id);
95
222
  }
223
+ export async function stopAllWatchersAndWait() {
224
+ stopAllWatchers();
225
+ await Promise.allSettled([...scanRuns.values()]);
226
+ }
227
+ export function getActiveWatcherScanCount() {
228
+ return scanRuns.size;
229
+ }
96
230
  export function getActiveWatcherCount() {
97
231
  return new Set([...watchers.keys(), ...pollers.keys()]).size;
98
232
  }
@@ -0,0 +1,4 @@
1
+ export declare function hardenWindowsPath(path: string, directory: boolean): void;
2
+ export declare function assertWindowsPathPrivate(path: string): void;
3
+ export declare function assertWindowsPathAncestorsTrusted(path: string): void;
4
+ export declare function assertWindowsExecutableTrusted(path: string): void;