pi-knowledge 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +23 -1
- package/index.ts +153 -34
- package/package.json +60 -60
- package/src/diagnostics/health.ts +11 -10
- package/src/embedding/provider.ts +30 -9
- package/src/embedding/vectors.ts +1 -1
- package/src/engine.ts +212 -85
- package/src/indexer/chunker.ts +115 -39
- package/src/indexer/chunkers/code-ast.ts +77 -17
- package/src/search/bm25.ts +19 -10
- package/src/search/fusion.ts +5 -4
- package/src/search/reranker.ts +25 -7
- package/src/search/vector.ts +10 -2
- package/src/storage/sqlite.ts +27 -8
- package/src/watcher/file-watcher.ts +92 -12
|
@@ -1,36 +1,116 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type Dirent, existsSync, type FSWatcher, readdirSync, statSync, watch } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
2
3
|
|
|
3
4
|
const watchers = new Map<string, FSWatcher>();
|
|
5
|
+
const pollers = new Map<string, ReturnType<typeof setInterval>>();
|
|
6
|
+
const snapshots = new Map<string, Map<string, string>>();
|
|
4
7
|
const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
5
8
|
const DEBOUNCE_MS = 2000;
|
|
9
|
+
const POLL_MS = 2000;
|
|
10
|
+
|
|
11
|
+
function scheduleUpdate(kbId: string, onUpdate: (kbId: string) => void): void {
|
|
12
|
+
const existing = debounceTimers.get(kbId);
|
|
13
|
+
if (existing) clearTimeout(existing);
|
|
14
|
+
debounceTimers.set(
|
|
15
|
+
kbId,
|
|
16
|
+
setTimeout(() => {
|
|
17
|
+
debounceTimers.delete(kbId);
|
|
18
|
+
onUpdate(kbId);
|
|
19
|
+
}, DEBOUNCE_MS),
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function scanSnapshot(dirPath: string): Map<string, string> {
|
|
24
|
+
const snapshot = new Map<string, string>();
|
|
25
|
+
|
|
26
|
+
function scan(dir: string): void {
|
|
27
|
+
let entries: Dirent[];
|
|
28
|
+
try {
|
|
29
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
30
|
+
} catch {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
for (const entry of entries) {
|
|
34
|
+
const fullPath = join(dir, entry.name);
|
|
35
|
+
try {
|
|
36
|
+
const stat = statSync(fullPath);
|
|
37
|
+
if (entry.isDirectory()) {
|
|
38
|
+
scan(fullPath);
|
|
39
|
+
} else if (entry.isFile()) {
|
|
40
|
+
snapshot.set(fullPath, `${stat.mtimeMs}:${stat.size}`);
|
|
41
|
+
}
|
|
42
|
+
} catch {
|
|
43
|
+
/* file disappeared or is unreadable */
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (existsSync(dirPath)) scan(dirPath);
|
|
49
|
+
return snapshot;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function snapshotsDiffer(a: Map<string, string>, b: Map<string, string>): boolean {
|
|
53
|
+
if (a.size !== b.size) return true;
|
|
54
|
+
for (const [path, value] of a) {
|
|
55
|
+
if (b.get(path) !== value) return true;
|
|
56
|
+
}
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function startPoller(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void {
|
|
61
|
+
snapshots.set(kbId, scanSnapshot(dirPath));
|
|
62
|
+
pollers.set(
|
|
63
|
+
kbId,
|
|
64
|
+
setInterval(() => {
|
|
65
|
+
const previous = snapshots.get(kbId) ?? new Map<string, string>();
|
|
66
|
+
const next = scanSnapshot(dirPath);
|
|
67
|
+
if (snapshotsDiffer(previous, next)) {
|
|
68
|
+
snapshots.set(kbId, next);
|
|
69
|
+
scheduleUpdate(kbId, onUpdate);
|
|
70
|
+
}
|
|
71
|
+
}, POLL_MS),
|
|
72
|
+
);
|
|
73
|
+
}
|
|
6
74
|
|
|
7
75
|
export function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void {
|
|
8
76
|
stopWatcher(kbId);
|
|
77
|
+
startPoller(kbId, dirPath, onUpdate);
|
|
9
78
|
try {
|
|
10
79
|
const watcher = watch(dirPath, { recursive: true }, () => {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
80
|
+
snapshots.set(kbId, scanSnapshot(dirPath));
|
|
81
|
+
scheduleUpdate(kbId, onUpdate);
|
|
82
|
+
});
|
|
83
|
+
watcher.on("error", () => {
|
|
84
|
+
watchers.get(kbId)?.close();
|
|
85
|
+
watchers.delete(kbId);
|
|
17
86
|
});
|
|
18
|
-
watcher.on("error", () => stopWatcher(kbId));
|
|
19
87
|
watchers.set(kbId, watcher);
|
|
20
|
-
} catch {
|
|
88
|
+
} catch {
|
|
89
|
+
/* polling fallback remains active */
|
|
90
|
+
}
|
|
21
91
|
}
|
|
22
92
|
|
|
23
93
|
export function stopWatcher(kbId: string): void {
|
|
24
94
|
watchers.get(kbId)?.close();
|
|
25
95
|
watchers.delete(kbId);
|
|
96
|
+
const poller = pollers.get(kbId);
|
|
97
|
+
if (poller) {
|
|
98
|
+
clearInterval(poller);
|
|
99
|
+
pollers.delete(kbId);
|
|
100
|
+
}
|
|
101
|
+
snapshots.delete(kbId);
|
|
26
102
|
const t = debounceTimers.get(kbId);
|
|
27
|
-
if (t) {
|
|
103
|
+
if (t) {
|
|
104
|
+
clearTimeout(t);
|
|
105
|
+
debounceTimers.delete(kbId);
|
|
106
|
+
}
|
|
28
107
|
}
|
|
29
108
|
|
|
30
109
|
export function stopAllWatchers(): void {
|
|
31
|
-
|
|
110
|
+
const ids = new Set([...watchers.keys(), ...pollers.keys()]);
|
|
111
|
+
for (const id of ids) stopWatcher(id);
|
|
32
112
|
}
|
|
33
113
|
|
|
34
114
|
export function getActiveWatcherCount(): number {
|
|
35
|
-
return watchers.size;
|
|
115
|
+
return new Set([...watchers.keys(), ...pollers.keys()]).size;
|
|
36
116
|
}
|