pi-knowledge 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +63 -24
- package/dist/index.d.ts +10 -4
- package/dist/index.js +300 -187
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +63 -9
- package/dist/src/engine.js +959 -170
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +10 -0
- package/dist/src/indexer/chunker.js +284 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +184 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +47 -0
- package/docs/configuration.md +42 -10
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { assertWindowsPathAncestorsTrusted, assertWindowsPathPrivate, hardenWindowsPath } from "./windows-security.js";
|
|
5
|
+
export function assertOwnedDirectory(path) {
|
|
6
|
+
const info = lstatSync(path);
|
|
7
|
+
if (info.isSymbolicLink() || !info.isDirectory()) {
|
|
8
|
+
throw new Error(`Expected a real directory owned by the current user: ${path}`);
|
|
9
|
+
}
|
|
10
|
+
if (process.platform === "win32") {
|
|
11
|
+
assertWindowsPathPrivate(path);
|
|
12
|
+
assertWindowsPathAncestorsTrusted(path);
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
if (typeof process.getuid === "function" && info.uid !== process.getuid()) {
|
|
16
|
+
throw new Error(`Directory is not owned by the current user: ${path}`);
|
|
17
|
+
}
|
|
18
|
+
if ((info.mode & 0o022) !== 0)
|
|
19
|
+
throw new Error(`Directory must not be group/world writable: ${path}`);
|
|
20
|
+
}
|
|
21
|
+
if (process.platform !== "win32" && typeof process.getuid === "function") {
|
|
22
|
+
const uid = process.getuid();
|
|
23
|
+
let current = realpathSync.native(path);
|
|
24
|
+
while (true) {
|
|
25
|
+
const ancestor = lstatSync(current);
|
|
26
|
+
if (ancestor.isSymbolicLink() || !ancestor.isDirectory()) {
|
|
27
|
+
throw new Error(`Path ancestor is not a real directory: ${current}`);
|
|
28
|
+
}
|
|
29
|
+
if (ancestor.uid !== uid && ancestor.uid !== 0) {
|
|
30
|
+
throw new Error(`Path ancestor is owned by an untrusted user: ${current}`);
|
|
31
|
+
}
|
|
32
|
+
if ((ancestor.mode & 0o022) !== 0 && (ancestor.mode & 0o1000) === 0) {
|
|
33
|
+
throw new Error(`Path ancestor permits untrusted rename operations: ${current}`);
|
|
34
|
+
}
|
|
35
|
+
const parent = dirname(current);
|
|
36
|
+
if (parent === current)
|
|
37
|
+
break;
|
|
38
|
+
current = parent;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export function ensurePrivateDirectory(path) {
|
|
43
|
+
mkdirSync(path, { recursive: true, mode: 0o700 });
|
|
44
|
+
hardenWindowsPath(path, true);
|
|
45
|
+
assertOwnedDirectory(path);
|
|
46
|
+
return realpathSync.native(path);
|
|
47
|
+
}
|
|
48
|
+
export function ensureOwnerOnlyDirectory(path) {
|
|
49
|
+
const canonical = ensurePrivateDirectory(path);
|
|
50
|
+
if (process.platform !== "win32") {
|
|
51
|
+
chmodSync(canonical, 0o700);
|
|
52
|
+
const info = statSync(canonical);
|
|
53
|
+
if ((info.mode & 0o077) !== 0)
|
|
54
|
+
throw new Error(`Directory must use owner-only permissions: ${canonical}`);
|
|
55
|
+
}
|
|
56
|
+
return canonical;
|
|
57
|
+
}
|
|
58
|
+
export function fsyncDirectory(path) {
|
|
59
|
+
try {
|
|
60
|
+
const fd = openSync(path, "r");
|
|
61
|
+
try {
|
|
62
|
+
fsyncSync(fd);
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
closeSync(fd);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Directory fsync is unavailable on some supported platforms.
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
export function prepareSafeNewFile(path) {
|
|
73
|
+
const requested = resolve(path);
|
|
74
|
+
const parentPath = realpathSync.native(dirname(requested));
|
|
75
|
+
assertOwnedDirectory(parentPath);
|
|
76
|
+
const outputPath = join(parentPath, basename(requested));
|
|
77
|
+
if (existsSync(outputPath))
|
|
78
|
+
throw new Error(`Refusing to overwrite existing file: ${outputPath}`);
|
|
79
|
+
const parent = statSync(parentPath);
|
|
80
|
+
return {
|
|
81
|
+
outputPath,
|
|
82
|
+
parentPath,
|
|
83
|
+
parentDevice: parent.dev,
|
|
84
|
+
parentInode: parent.ino,
|
|
85
|
+
tempPath: join(parentPath, `.${basename(requested)}.${randomUUID()}.tmp`),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function publishNewFile(tempPath, target) {
|
|
89
|
+
const currentParent = statSync(target.parentPath);
|
|
90
|
+
if (currentParent.dev !== target.parentDevice || currentParent.ino !== target.parentInode) {
|
|
91
|
+
throw new Error(`Export directory changed during publication: ${target.parentPath}`);
|
|
92
|
+
}
|
|
93
|
+
linkSync(tempPath, target.outputPath);
|
|
94
|
+
fsyncDirectory(target.parentPath);
|
|
95
|
+
rmSync(tempPath, { force: true });
|
|
96
|
+
}
|
|
97
|
+
export function hardenPrivateFile(path, label) {
|
|
98
|
+
const info = lstatSync(path);
|
|
99
|
+
if (info.isSymbolicLink() || !info.isFile())
|
|
100
|
+
throw new Error(`Invalid ${label}: ${path}`);
|
|
101
|
+
if (process.platform === "win32") {
|
|
102
|
+
hardenWindowsPath(path, false);
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
if (typeof process.getuid === "function" && info.uid !== process.getuid()) {
|
|
106
|
+
throw new Error(`${label} is not owned by the current user: ${path}`);
|
|
107
|
+
}
|
|
108
|
+
chmodSync(path, 0o600);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
export function readPrivateJson(path, label) {
|
|
112
|
+
assertOwnedDirectory(dirname(path));
|
|
113
|
+
const direct = lstatSync(path);
|
|
114
|
+
if (direct.isSymbolicLink() || !direct.isFile())
|
|
115
|
+
throw new Error(`Invalid ${label}: ${path}`);
|
|
116
|
+
if (process.platform === "win32") {
|
|
117
|
+
assertWindowsPathPrivate(path);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
if (typeof process.getuid === "function" && direct.uid !== process.getuid()) {
|
|
121
|
+
throw new Error(`${label} is not owned by the current user: ${path}`);
|
|
122
|
+
}
|
|
123
|
+
if ((direct.mode & 0o077) !== 0)
|
|
124
|
+
throw new Error(`${label} permissions are too broad: ${path}`);
|
|
125
|
+
}
|
|
126
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
127
|
+
try {
|
|
128
|
+
const opened = fstatSync(fd);
|
|
129
|
+
if (opened.dev !== direct.dev || opened.ino !== direct.ino)
|
|
130
|
+
throw new Error(`${label} changed while opening: ${path}`);
|
|
131
|
+
return JSON.parse(readFileSync(fd, "utf8"));
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
closeSync(fd);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
export function writeJsonAtomic(path, value) {
|
|
138
|
+
const parent = ensurePrivateDirectory(dirname(path));
|
|
139
|
+
const temp = join(parent, `.${randomUUID()}.tmp`);
|
|
140
|
+
let fd;
|
|
141
|
+
try {
|
|
142
|
+
fd = openSync(temp, "wx", 0o600);
|
|
143
|
+
writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
144
|
+
fsyncSync(fd);
|
|
145
|
+
closeSync(fd);
|
|
146
|
+
fd = undefined;
|
|
147
|
+
renameSync(temp, path);
|
|
148
|
+
hardenWindowsPath(path, false);
|
|
149
|
+
fsyncDirectory(parent);
|
|
150
|
+
}
|
|
151
|
+
finally {
|
|
152
|
+
if (fd !== undefined)
|
|
153
|
+
closeSync(fd);
|
|
154
|
+
rmSync(temp, { force: true });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
export function writeJsonExclusive(path, value) {
|
|
158
|
+
const parent = ensurePrivateDirectory(dirname(path));
|
|
159
|
+
const temp = join(parent, `.${randomUUID()}.tmp`);
|
|
160
|
+
let fd;
|
|
161
|
+
try {
|
|
162
|
+
fd = openSync(temp, "wx", 0o600);
|
|
163
|
+
writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
164
|
+
fsyncSync(fd);
|
|
165
|
+
closeSync(fd);
|
|
166
|
+
fd = undefined;
|
|
167
|
+
try {
|
|
168
|
+
linkSync(temp, path);
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (error.code === "EEXIST")
|
|
172
|
+
return false;
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
hardenPrivateFile(path, "knowledge profile registry");
|
|
176
|
+
fsyncDirectory(parent);
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
if (fd !== undefined)
|
|
181
|
+
closeSync(fd);
|
|
182
|
+
rmSync(temp, { force: true });
|
|
183
|
+
}
|
|
184
|
+
}
|
package/dist/src/status.js
CHANGED
|
@@ -48,11 +48,16 @@ export function formatKnowledgeStatus(options) {
|
|
|
48
48
|
if (diagnostic.job.error_message)
|
|
49
49
|
lines.push(` error: ${diagnostic.job.error_message}`);
|
|
50
50
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
if (diagnostic.scan_error)
|
|
52
|
+
lines.push(` ⚠️ source scan failed: ${diagnostic.scan_error}`);
|
|
53
|
+
else
|
|
54
|
+
lines.push(` coverage: ${diagnostic.coverage_percent}% (${diagnostic.indexed_files}/${diagnostic.total_source_files} files)`);
|
|
55
|
+
const staleCount = diagnostic.stale_file_count ?? diagnostic.stale_files.length;
|
|
56
|
+
const orphanCount = diagnostic.orphan_file_count ?? diagnostic.orphan_files.length;
|
|
57
|
+
if (staleCount > 0)
|
|
58
|
+
lines.push(` ⚠️ stale: ${staleCount} files modified since last index`);
|
|
59
|
+
if (orphanCount > 0)
|
|
60
|
+
lines.push(` ⚠️ orphans: ${orphanCount} chunks reference deleted files`);
|
|
56
61
|
if (diagnostic.stuck_indexing)
|
|
57
62
|
lines.push(` ⚠️ indexing appears stuck for ${Math.round(diagnostic.last_progress_age_ms / 60_000)}m; remove and rebuild if no pi process is actively indexing it`);
|
|
58
63
|
if (diagnostic.skipped_files.total > 0)
|
|
@@ -61,8 +66,8 @@ export function formatKnowledgeStatus(options) {
|
|
|
61
66
|
.map(([reason, count]) => `${reason}: ${count}`)
|
|
62
67
|
.join(", ")})`);
|
|
63
68
|
}
|
|
64
|
-
const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.stale_files.length, 0);
|
|
65
|
-
const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + diagnostic.orphan_files.length, 0);
|
|
69
|
+
const totalStale = snapshot.diagnostics.reduce((count, diagnostic) => count + (diagnostic.stale_file_count ?? diagnostic.stale_files.length), 0);
|
|
70
|
+
const totalOrphans = snapshot.diagnostics.reduce((count, diagnostic) => count + (diagnostic.orphan_file_count ?? diagnostic.orphan_files.length), 0);
|
|
66
71
|
const totalStuck = snapshot.diagnostics.filter((diagnostic) => diagnostic.stuck_indexing).length;
|
|
67
72
|
const totalIndexing = snapshot.knowledge_bases.filter((kb) => kb.status === "indexing" && !diagnosticsByKb.get(kb.id)?.stuck_indexing).length;
|
|
68
73
|
const totalErrors = snapshot.knowledge_bases.filter((kb) => kb.status === "error").length;
|
|
@@ -8,6 +8,7 @@ export interface KnowledgeBase {
|
|
|
8
8
|
source_options: string | null;
|
|
9
9
|
created_at: number;
|
|
10
10
|
updated_at: number;
|
|
11
|
+
indexed_at: number;
|
|
11
12
|
chunk_count: number;
|
|
12
13
|
file_count: number;
|
|
13
14
|
embedding_model: string;
|
|
@@ -83,10 +84,11 @@ export declare function createKB(db: Database.Database, opts: {
|
|
|
83
84
|
}): KnowledgeBase;
|
|
84
85
|
export declare function getKB(db: Database.Database, id: string): KnowledgeBase | undefined;
|
|
85
86
|
export declare function getKBByName(db: Database.Database, name: string): KnowledgeBase | undefined;
|
|
86
|
-
export declare function listKBs(db: Database.Database): KnowledgeBase[];
|
|
87
|
+
export declare function listKBs(db: Database.Database, limit?: number): KnowledgeBase[];
|
|
87
88
|
export declare function deleteKB(db: Database.Database, id: string): void;
|
|
88
89
|
export declare function updateKBStatus(db: Database.Database, id: string, status: KnowledgeBase["status"]): void;
|
|
89
90
|
export declare function updateKBCounts(db: Database.Database, id: string, chunkCount: number, fileCount: number): void;
|
|
91
|
+
export declare function markKBIndexed(db: Database.Database, id: string): void;
|
|
90
92
|
export declare function updateKBSourceOptions(db: Database.Database, id: string, sourceOptions: string): void;
|
|
91
93
|
export declare function updateKBEmbeddingMetadata(db: Database.Database, id: string, embeddingModel: string, embeddingSignature: string | null, embeddingDimension: number | null): void;
|
|
92
94
|
export declare function startIndexingJob(db: Database.Database, kbId: string, operation: IndexingJob["operation"], message: string): void;
|
|
@@ -123,6 +125,7 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
|
|
|
123
125
|
export declare function getChunkCount(db: Database.Database, kbId: string): number;
|
|
124
126
|
export declare function getFileCount(db: Database.Database, kbId: string): number;
|
|
125
127
|
export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
|
|
128
|
+
export declare function getSymbolsByFile(db: Database.Database, kbId: string, filePath: string): KnowledgeSymbolInsert[];
|
|
126
129
|
export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
|
|
127
130
|
export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
|
|
128
131
|
export declare function searchSymbols(db: Database.Database, query: string, options?: {
|
|
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
|
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
|
-
const SCHEMA_VERSION =
|
|
7
|
+
const SCHEMA_VERSION = 6;
|
|
8
8
|
const ITERATION_BATCH_SIZE = 500;
|
|
9
9
|
const SCHEMA_SQL = `
|
|
10
10
|
CREATE TABLE IF NOT EXISTS schema_version (
|
|
@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases (
|
|
|
20
20
|
source_options TEXT,
|
|
21
21
|
created_at INTEGER NOT NULL,
|
|
22
22
|
updated_at INTEGER NOT NULL,
|
|
23
|
+
indexed_at INTEGER NOT NULL DEFAULT 0,
|
|
23
24
|
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
24
25
|
file_count INTEGER NOT NULL DEFAULT 0,
|
|
25
26
|
embedding_model TEXT NOT NULL DEFAULT 'multilingual-e5-small',
|
|
@@ -316,6 +317,30 @@ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
|
|
|
316
317
|
}
|
|
317
318
|
continue;
|
|
318
319
|
}
|
|
320
|
+
if (v === 6) {
|
|
321
|
+
const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all();
|
|
322
|
+
if (!columns.some((column) => column.name === "indexed_at")) {
|
|
323
|
+
db.exec("ALTER TABLE knowledge_bases ADD COLUMN indexed_at INTEGER NOT NULL DEFAULT 0");
|
|
324
|
+
db.exec(`
|
|
325
|
+
UPDATE knowledge_bases
|
|
326
|
+
SET indexed_at = COALESCE(
|
|
327
|
+
(
|
|
328
|
+
SELECT updated_at
|
|
329
|
+
FROM indexing_jobs
|
|
330
|
+
WHERE indexing_jobs.kb_id = knowledge_bases.id
|
|
331
|
+
AND indexing_jobs.status = 'succeeded'
|
|
332
|
+
),
|
|
333
|
+
(
|
|
334
|
+
SELECT MAX(indexed_at)
|
|
335
|
+
FROM chunks
|
|
336
|
+
WHERE chunks.kb_id = knowledge_bases.id
|
|
337
|
+
),
|
|
338
|
+
0
|
|
339
|
+
)
|
|
340
|
+
`);
|
|
341
|
+
}
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
319
344
|
if (migrations[v])
|
|
320
345
|
db.exec(migrations[v]);
|
|
321
346
|
}
|
|
@@ -340,8 +365,13 @@ export function getKB(db, id) {
|
|
|
340
365
|
export function getKBByName(db, name) {
|
|
341
366
|
return db.prepare("SELECT * FROM knowledge_bases WHERE name = ?").get(name);
|
|
342
367
|
}
|
|
343
|
-
export function listKBs(db) {
|
|
344
|
-
|
|
368
|
+
export function listKBs(db, limit) {
|
|
369
|
+
if (limit === undefined) {
|
|
370
|
+
return db.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC").all();
|
|
371
|
+
}
|
|
372
|
+
return db
|
|
373
|
+
.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC LIMIT ?")
|
|
374
|
+
.all(Math.max(0, Math.trunc(limit)));
|
|
345
375
|
}
|
|
346
376
|
export function deleteKB(db, id) {
|
|
347
377
|
db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
|
|
@@ -355,6 +385,10 @@ export function updateKBStatus(db, id, status) {
|
|
|
355
385
|
export function updateKBCounts(db, id, chunkCount, fileCount) {
|
|
356
386
|
db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(chunkCount, fileCount, Date.now(), id);
|
|
357
387
|
}
|
|
388
|
+
export function markKBIndexed(db, id) {
|
|
389
|
+
const now = Date.now();
|
|
390
|
+
db.prepare("UPDATE knowledge_bases SET indexed_at = ?, updated_at = ? WHERE id = ?").run(now, now, id);
|
|
391
|
+
}
|
|
358
392
|
export function updateKBSourceOptions(db, id, sourceOptions) {
|
|
359
393
|
db.prepare("UPDATE knowledge_bases SET source_options = ?, updated_at = ? WHERE id = ?").run(sourceOptions, Date.now(), id);
|
|
360
394
|
}
|
|
@@ -562,6 +596,15 @@ export function insertSymbols(db, kbId, symbols) {
|
|
|
562
596
|
});
|
|
563
597
|
insertMany(symbols);
|
|
564
598
|
}
|
|
599
|
+
export function getSymbolsByFile(db, kbId, filePath) {
|
|
600
|
+
return db
|
|
601
|
+
.prepare(`SELECT name, kind, file_path, file_type, start_line, end_line,
|
|
602
|
+
container_name, signature, text, metadata_json
|
|
603
|
+
FROM symbols
|
|
604
|
+
WHERE kb_id = ? AND file_path = ?
|
|
605
|
+
ORDER BY indexed_at, rowid`)
|
|
606
|
+
.all(kbId, filePath);
|
|
607
|
+
}
|
|
565
608
|
export function deleteSymbolsByKB(db, kbId) {
|
|
566
609
|
db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
|
|
567
610
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isObjectRecord(value: unknown): value is Record<string, unknown>;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { type ScanOptions } from "../indexer/chunker.ts";
|
|
2
|
-
export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void
|
|
2
|
+
export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): Promise<void>;
|
|
3
3
|
export declare function stopWatcher(kbId: string): void;
|
|
4
4
|
export declare function stopAllWatchers(): void;
|
|
5
|
+
export declare function stopAllWatchersAndWait(): Promise<void>;
|
|
6
|
+
export declare function getActiveWatcherScanCount(): number;
|
|
5
7
|
export declare function getActiveWatcherCount(): number;
|
|
@@ -1,11 +1,24 @@
|
|
|
1
1
|
import { existsSync, statSync, watch } from "node:fs";
|
|
2
|
-
import {
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
|
|
4
|
+
import { createSkippedScanStats, iterateScannableFileVisitsAsync } from "../indexer/chunker.js";
|
|
3
5
|
const watchers = new Map();
|
|
4
6
|
const pollers = new Map();
|
|
5
7
|
const snapshots = new Map();
|
|
6
8
|
const debounceTimers = new Map();
|
|
9
|
+
const scanRuns = new Map();
|
|
10
|
+
const scanControllers = new Map();
|
|
11
|
+
const scanGenerations = new Map();
|
|
12
|
+
const scanRequested = new Set();
|
|
13
|
+
const eventPaths = new Map();
|
|
14
|
+
const baselineEvents = new Set();
|
|
15
|
+
const baselineRootStamps = new Map();
|
|
16
|
+
let scanPermit = Promise.resolve();
|
|
17
|
+
const generations = new Map();
|
|
7
18
|
const DEBOUNCE_MS = 2000;
|
|
8
19
|
const POLL_MS = 2000;
|
|
20
|
+
const SCAN_YIELD_ENTRIES = 50;
|
|
21
|
+
const MAX_EVENT_PATHS = 1_024;
|
|
9
22
|
function scheduleUpdate(kbId, onUpdate) {
|
|
10
23
|
const existing = debounceTimers.get(kbId);
|
|
11
24
|
if (existing)
|
|
@@ -15,56 +28,163 @@ function scheduleUpdate(kbId, onUpdate) {
|
|
|
15
28
|
onUpdate(kbId);
|
|
16
29
|
}, DEBOUNCE_MS));
|
|
17
30
|
}
|
|
18
|
-
function
|
|
31
|
+
function rootStamp(path) {
|
|
32
|
+
try {
|
|
33
|
+
const info = statSync(path);
|
|
34
|
+
return `${info.mtimeMs}:${info.ctimeMs}:${info.size}`;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function scanSnapshot(dirPath, options = {}, signal) {
|
|
19
41
|
const snapshot = new Map();
|
|
20
42
|
if (!existsSync(dirPath))
|
|
21
43
|
return snapshot;
|
|
44
|
+
if (signal?.aborted)
|
|
45
|
+
return snapshot;
|
|
22
46
|
const skipped = createSkippedScanStats();
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
snapshot
|
|
47
|
+
let processed = 0;
|
|
48
|
+
for await (const file of iterateScannableFileVisitsAsync(dirPath, skipped, options, signal)) {
|
|
49
|
+
if (signal?.aborted)
|
|
50
|
+
return snapshot;
|
|
51
|
+
if (file) {
|
|
52
|
+
snapshot.set(file.path, `${file.mtimeMs}:${file.ctimeMs}:${file.size}:${file.inode}`);
|
|
27
53
|
}
|
|
28
|
-
|
|
29
|
-
|
|
54
|
+
processed++;
|
|
55
|
+
if (processed % SCAN_YIELD_ENTRIES === 0) {
|
|
56
|
+
await yieldToEventLoop();
|
|
57
|
+
if (signal?.aborted)
|
|
58
|
+
return snapshot;
|
|
30
59
|
}
|
|
31
60
|
}
|
|
32
61
|
return snapshot;
|
|
33
62
|
}
|
|
34
|
-
function
|
|
35
|
-
|
|
63
|
+
async function withScanPermit(operation) {
|
|
64
|
+
const previous = scanPermit;
|
|
65
|
+
let release = () => { };
|
|
66
|
+
scanPermit = new Promise((resolve) => {
|
|
67
|
+
release = resolve;
|
|
68
|
+
});
|
|
69
|
+
await previous.catch(() => { });
|
|
70
|
+
try {
|
|
71
|
+
return await operation();
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
release();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function snapshotsDiffer(left, right) {
|
|
78
|
+
if (left.size !== right.size)
|
|
79
|
+
return true;
|
|
80
|
+
for (const [path, value] of left) {
|
|
81
|
+
if (right.get(path) !== value)
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
function eventTouchesSnapshot(path, snapshot) {
|
|
87
|
+
if (path === undefined)
|
|
36
88
|
return true;
|
|
37
|
-
|
|
38
|
-
|
|
89
|
+
if (!snapshot)
|
|
90
|
+
return false;
|
|
91
|
+
if (snapshot.has(path))
|
|
92
|
+
return true;
|
|
93
|
+
const prefix = `${path}/`;
|
|
94
|
+
for (const candidate of snapshot.keys()) {
|
|
95
|
+
if (candidate.startsWith(prefix))
|
|
39
96
|
return true;
|
|
40
97
|
}
|
|
41
98
|
return false;
|
|
42
99
|
}
|
|
43
|
-
function
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (
|
|
100
|
+
function requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive = true) {
|
|
101
|
+
if (generations.get(kbId) !== generation)
|
|
102
|
+
return Promise.resolve();
|
|
103
|
+
const existing = scanRuns.get(kbId);
|
|
104
|
+
if (existing) {
|
|
105
|
+
if (scanGenerations.get(kbId) === generation) {
|
|
106
|
+
if (followUpIfActive)
|
|
107
|
+
scanRequested.add(kbId);
|
|
108
|
+
return existing;
|
|
109
|
+
}
|
|
110
|
+
return existing.then(() => requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive), () => requestScan(kbId, generation, dirPath, onUpdate, options, followUpIfActive));
|
|
111
|
+
}
|
|
112
|
+
const controller = new AbortController();
|
|
113
|
+
const run = (async () => {
|
|
114
|
+
do {
|
|
115
|
+
if (generations.get(kbId) !== generation)
|
|
116
|
+
return;
|
|
117
|
+
scanRequested.delete(kbId);
|
|
118
|
+
const next = await withScanPermit(() => scanSnapshot(dirPath, options, controller.signal));
|
|
119
|
+
if (controller.signal.aborted)
|
|
120
|
+
return;
|
|
121
|
+
if (generations.get(kbId) !== generation)
|
|
122
|
+
return;
|
|
123
|
+
const previous = snapshots.get(kbId);
|
|
49
124
|
snapshots.set(kbId, next);
|
|
50
|
-
|
|
125
|
+
const baselineStamp = baselineRootStamps.get(kbId);
|
|
126
|
+
baselineRootStamps.delete(kbId);
|
|
127
|
+
const baselineChanged = previous === undefined && baselineStamp !== undefined && rootStamp(dirPath) !== baselineStamp;
|
|
128
|
+
const events = eventPaths.get(kbId);
|
|
129
|
+
eventPaths.delete(kbId);
|
|
130
|
+
const eventRelevant = events !== undefined &&
|
|
131
|
+
[...events].some((path) => eventTouchesSnapshot(path, previous) || eventTouchesSnapshot(path, next));
|
|
132
|
+
if (baselineEvents.delete(kbId) ||
|
|
133
|
+
baselineChanged ||
|
|
134
|
+
(previous && snapshotsDiffer(previous, next)) ||
|
|
135
|
+
eventRelevant) {
|
|
136
|
+
scheduleUpdate(kbId, onUpdate);
|
|
137
|
+
}
|
|
138
|
+
} while (scanRequested.has(kbId));
|
|
139
|
+
})().finally(() => {
|
|
140
|
+
if (scanRuns.get(kbId) === run) {
|
|
141
|
+
scanRuns.delete(kbId);
|
|
142
|
+
scanControllers.delete(kbId);
|
|
143
|
+
scanGenerations.delete(kbId);
|
|
51
144
|
}
|
|
52
|
-
}
|
|
145
|
+
});
|
|
146
|
+
scanRuns.set(kbId, run);
|
|
147
|
+
scanControllers.set(kbId, controller);
|
|
148
|
+
scanGenerations.set(kbId, generation);
|
|
149
|
+
void run.catch(() => {
|
|
150
|
+
/* transient scan errors are retried by the polling fallback */
|
|
151
|
+
});
|
|
152
|
+
return run;
|
|
153
|
+
}
|
|
154
|
+
function startPoller(kbId, generation, dirPath, onUpdate, options = {}) {
|
|
155
|
+
const initialScan = requestScan(kbId, generation, dirPath, onUpdate, options, false);
|
|
156
|
+
pollers.set(kbId, setInterval(() => requestScan(kbId, generation, dirPath, onUpdate, options, false), POLL_MS));
|
|
157
|
+
return initialScan;
|
|
53
158
|
}
|
|
54
159
|
export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
|
|
55
160
|
stopWatcher(kbId);
|
|
56
|
-
|
|
161
|
+
baselineRootStamps.set(kbId, rootStamp(dirPath));
|
|
162
|
+
const generation = {};
|
|
163
|
+
generations.set(kbId, generation);
|
|
164
|
+
const initialScan = startPoller(kbId, generation, dirPath, onUpdate, options);
|
|
57
165
|
try {
|
|
58
|
-
const watcher = watch(dirPath, { recursive: true }, () => {
|
|
59
|
-
|
|
60
|
-
const next = scanSnapshot(dirPath, options);
|
|
61
|
-
if (!snapshotsDiffer(previous, next))
|
|
166
|
+
const watcher = watch(dirPath, { recursive: true }, (_event, filename) => {
|
|
167
|
+
if (generations.get(kbId) !== generation)
|
|
62
168
|
return;
|
|
63
|
-
snapshots.
|
|
64
|
-
|
|
169
|
+
if (!snapshots.has(kbId))
|
|
170
|
+
baselineEvents.add(kbId);
|
|
171
|
+
const paths = eventPaths.get(kbId) ?? new Set();
|
|
172
|
+
if (!paths.has(undefined)) {
|
|
173
|
+
if (filename === null || paths.size >= MAX_EVENT_PATHS) {
|
|
174
|
+
paths.clear();
|
|
175
|
+
paths.add(undefined);
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
paths.add(resolve(dirPath, filename.toString()));
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
eventPaths.set(kbId, paths);
|
|
182
|
+
void requestScan(kbId, generation, dirPath, onUpdate, options);
|
|
65
183
|
});
|
|
66
184
|
watcher.on("error", () => {
|
|
67
|
-
watchers.get(kbId)
|
|
185
|
+
if (watchers.get(kbId) !== watcher || generations.get(kbId) !== generation)
|
|
186
|
+
return;
|
|
187
|
+
watcher.close();
|
|
68
188
|
watchers.delete(kbId);
|
|
69
189
|
});
|
|
70
190
|
watchers.set(kbId, watcher);
|
|
@@ -72,8 +192,11 @@ export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
|
|
|
72
192
|
catch {
|
|
73
193
|
/* polling fallback remains active */
|
|
74
194
|
}
|
|
195
|
+
return initialScan;
|
|
75
196
|
}
|
|
76
197
|
export function stopWatcher(kbId) {
|
|
198
|
+
generations.delete(kbId);
|
|
199
|
+
scanControllers.get(kbId)?.abort();
|
|
77
200
|
watchers.get(kbId)?.close();
|
|
78
201
|
watchers.delete(kbId);
|
|
79
202
|
const poller = pollers.get(kbId);
|
|
@@ -81,18 +204,29 @@ export function stopWatcher(kbId) {
|
|
|
81
204
|
clearInterval(poller);
|
|
82
205
|
pollers.delete(kbId);
|
|
83
206
|
}
|
|
207
|
+
scanRequested.delete(kbId);
|
|
84
208
|
snapshots.delete(kbId);
|
|
85
|
-
const
|
|
86
|
-
if (
|
|
87
|
-
clearTimeout(
|
|
209
|
+
const timer = debounceTimers.get(kbId);
|
|
210
|
+
if (timer) {
|
|
211
|
+
clearTimeout(timer);
|
|
88
212
|
debounceTimers.delete(kbId);
|
|
89
213
|
}
|
|
214
|
+
eventPaths.delete(kbId);
|
|
215
|
+
baselineRootStamps.delete(kbId);
|
|
216
|
+
baselineEvents.delete(kbId);
|
|
90
217
|
}
|
|
91
218
|
export function stopAllWatchers() {
|
|
92
|
-
const ids = new Set([...watchers.keys(), ...pollers.keys()]);
|
|
219
|
+
const ids = new Set([...watchers.keys(), ...pollers.keys(), ...scanRuns.keys()]);
|
|
93
220
|
for (const id of ids)
|
|
94
221
|
stopWatcher(id);
|
|
95
222
|
}
|
|
223
|
+
export async function stopAllWatchersAndWait() {
|
|
224
|
+
stopAllWatchers();
|
|
225
|
+
await Promise.allSettled([...scanRuns.values()]);
|
|
226
|
+
}
|
|
227
|
+
export function getActiveWatcherScanCount() {
|
|
228
|
+
return scanRuns.size;
|
|
229
|
+
}
|
|
96
230
|
export function getActiveWatcherCount() {
|
|
97
231
|
return new Set([...watchers.keys(), ...pollers.keys()]).size;
|
|
98
232
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function hardenWindowsPath(path: string, directory: boolean): void;
|
|
2
|
+
export declare function assertWindowsPathPrivate(path: string): void;
|
|
3
|
+
export declare function assertWindowsPathAncestorsTrusted(path: string): void;
|
|
4
|
+
export declare function assertWindowsExecutableTrusted(path: string): void;
|