pi-knowledge 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +4 -2
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +9 -0
- package/package.json +10 -5
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { basename, dirname, join } from "node:path";
|
|
5
|
+
import Database from "better-sqlite3";
|
|
6
|
+
const SCHEMA_VERSION = 3;
|
|
7
|
+
const SCHEMA_SQL = `
|
|
8
|
+
CREATE TABLE IF NOT EXISTS schema_version (
|
|
9
|
+
version INTEGER NOT NULL
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
CREATE TABLE IF NOT EXISTS knowledge_bases (
|
|
13
|
+
id TEXT PRIMARY KEY,
|
|
14
|
+
name TEXT NOT NULL,
|
|
15
|
+
description TEXT,
|
|
16
|
+
source_path TEXT,
|
|
17
|
+
source_type TEXT NOT NULL DEFAULT 'directory',
|
|
18
|
+
source_options TEXT,
|
|
19
|
+
created_at INTEGER NOT NULL,
|
|
20
|
+
updated_at INTEGER NOT NULL,
|
|
21
|
+
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
22
|
+
file_count INTEGER NOT NULL DEFAULT 0,
|
|
23
|
+
embedding_model TEXT NOT NULL DEFAULT 'multilingual-e5-small',
|
|
24
|
+
status TEXT NOT NULL DEFAULT 'ready'
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
28
|
+
id TEXT PRIMARY KEY,
|
|
29
|
+
kb_id TEXT NOT NULL,
|
|
30
|
+
content_hash TEXT NOT NULL,
|
|
31
|
+
content TEXT NOT NULL,
|
|
32
|
+
content_tokenized TEXT NOT NULL,
|
|
33
|
+
file_path TEXT NOT NULL,
|
|
34
|
+
file_type TEXT NOT NULL DEFAULT 'text',
|
|
35
|
+
start_line INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
end_line INTEGER NOT NULL DEFAULT 0,
|
|
37
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
38
|
+
indexed_at INTEGER NOT NULL,
|
|
39
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
43
|
+
kb_id TEXT PRIMARY KEY,
|
|
44
|
+
operation TEXT NOT NULL,
|
|
45
|
+
status TEXT NOT NULL,
|
|
46
|
+
phase TEXT NOT NULL,
|
|
47
|
+
message TEXT NOT NULL,
|
|
48
|
+
started_at INTEGER NOT NULL,
|
|
49
|
+
updated_at INTEGER NOT NULL,
|
|
50
|
+
last_progress_at INTEGER NOT NULL,
|
|
51
|
+
processed_files INTEGER NOT NULL DEFAULT 0,
|
|
52
|
+
processed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
53
|
+
total_files INTEGER,
|
|
54
|
+
skipped_total INTEGER NOT NULL DEFAULT 0,
|
|
55
|
+
added_chunks INTEGER NOT NULL DEFAULT 0,
|
|
56
|
+
removed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
57
|
+
unchanged_chunks INTEGER NOT NULL DEFAULT 0,
|
|
58
|
+
error_message TEXT,
|
|
59
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
|
|
64
|
+
|
|
65
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
66
|
+
content_tokenized,
|
|
67
|
+
content=chunks,
|
|
68
|
+
content_rowid=rowid
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
-- Triggers to keep FTS in sync with chunks table
|
|
72
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
|
|
73
|
+
INSERT INTO chunks_fts(rowid, content_tokenized) VALUES (new.rowid, new.content_tokenized);
|
|
74
|
+
END;
|
|
75
|
+
|
|
76
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
77
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, content_tokenized) VALUES('delete', old.rowid, old.content_tokenized);
|
|
78
|
+
END;
|
|
79
|
+
|
|
80
|
+
CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
|
|
81
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, content_tokenized) VALUES('delete', old.rowid, old.content_tokenized);
|
|
82
|
+
INSERT INTO chunks_fts(rowid, content_tokenized) VALUES (new.rowid, new.content_tokenized);
|
|
83
|
+
END;
|
|
84
|
+
`;
|
|
85
|
+
export function getDefaultKnowledgeDir() {
|
|
86
|
+
const explicit = process.env.PI_KNOWLEDGE_DIR?.trim() || process.env.OMP_KNOWLEDGE_DIR?.trim();
|
|
87
|
+
if (explicit)
|
|
88
|
+
return explicit;
|
|
89
|
+
const configuredAgentDir = process.env.PI_CODING_AGENT_DIR?.trim() || process.env.OMP_CODING_AGENT_DIR?.trim();
|
|
90
|
+
if (configuredAgentDir)
|
|
91
|
+
return resolveHostKnowledgeDir(dirname(configuredAgentDir));
|
|
92
|
+
const hostRoot = join(homedir(), isOmpHost() ? ".omp" : ".pi");
|
|
93
|
+
return resolveHostKnowledgeDir(hostRoot);
|
|
94
|
+
}
|
|
95
|
+
export function resolveHostKnowledgeDir(hostRoot, options = {}) {
|
|
96
|
+
const pathExists = options.exists ?? existsSync;
|
|
97
|
+
const target = join(hostRoot, "knowledge");
|
|
98
|
+
const legacyPiDir = options.legacyPiDir ?? join(homedir(), ".pi", "knowledge");
|
|
99
|
+
if (basename(hostRoot) === ".omp" &&
|
|
100
|
+
hostRoot.startsWith(homedir()) &&
|
|
101
|
+
!pathExists(target) &&
|
|
102
|
+
pathExists(legacyPiDir)) {
|
|
103
|
+
return legacyPiDir;
|
|
104
|
+
}
|
|
105
|
+
return target;
|
|
106
|
+
}
|
|
107
|
+
function isOmpHost() {
|
|
108
|
+
if (process.env.OMP_PROFILE?.trim())
|
|
109
|
+
return true;
|
|
110
|
+
const candidates = [process.argv[1], process.execPath].filter((value) => typeof value === "string");
|
|
111
|
+
return candidates.some((value) => basename(value).toLowerCase() === "omp");
|
|
112
|
+
}
|
|
113
|
+
export function openDatabase(knowledgeDir) {
|
|
114
|
+
const dir = knowledgeDir ?? getDefaultKnowledgeDir();
|
|
115
|
+
if (!existsSync(dir)) {
|
|
116
|
+
mkdirSync(dir, { recursive: true });
|
|
117
|
+
}
|
|
118
|
+
const dbPath = join(dir, "knowledge.db");
|
|
119
|
+
const db = new Database(dbPath);
|
|
120
|
+
db.pragma("journal_mode = WAL");
|
|
121
|
+
db.pragma("busy_timeout = 5000");
|
|
122
|
+
db.pragma("foreign_keys = ON");
|
|
123
|
+
const hasVersion = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'").get();
|
|
124
|
+
if (!hasVersion) {
|
|
125
|
+
db.exec(SCHEMA_SQL);
|
|
126
|
+
db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
const row = db.prepare("SELECT version FROM schema_version").get();
|
|
130
|
+
const currentVersion = row?.version ?? 0;
|
|
131
|
+
if (currentVersion < SCHEMA_VERSION) {
|
|
132
|
+
runMigrations(db, currentVersion, SCHEMA_VERSION);
|
|
133
|
+
db.prepare("UPDATE schema_version SET version = ?").run(SCHEMA_VERSION);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return db;
|
|
137
|
+
}
|
|
138
|
+
function runMigrations(db, from, to) {
|
|
139
|
+
const migrations = {
|
|
140
|
+
2: `
|
|
141
|
+
CREATE TABLE IF NOT EXISTS indexing_jobs (
|
|
142
|
+
kb_id TEXT PRIMARY KEY,
|
|
143
|
+
operation TEXT NOT NULL,
|
|
144
|
+
status TEXT NOT NULL,
|
|
145
|
+
phase TEXT NOT NULL,
|
|
146
|
+
message TEXT NOT NULL,
|
|
147
|
+
started_at INTEGER NOT NULL,
|
|
148
|
+
updated_at INTEGER NOT NULL,
|
|
149
|
+
last_progress_at INTEGER NOT NULL,
|
|
150
|
+
processed_files INTEGER NOT NULL DEFAULT 0,
|
|
151
|
+
processed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
152
|
+
total_files INTEGER,
|
|
153
|
+
skipped_total INTEGER NOT NULL DEFAULT 0,
|
|
154
|
+
added_chunks INTEGER NOT NULL DEFAULT 0,
|
|
155
|
+
removed_chunks INTEGER NOT NULL DEFAULT 0,
|
|
156
|
+
unchanged_chunks INTEGER NOT NULL DEFAULT 0,
|
|
157
|
+
error_message TEXT,
|
|
158
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
159
|
+
);
|
|
160
|
+
`,
|
|
161
|
+
};
|
|
162
|
+
for (let v = from + 1; v <= to; v++) {
|
|
163
|
+
if (v === 3) {
|
|
164
|
+
const columns = db.prepare("PRAGMA table_info(knowledge_bases)").all();
|
|
165
|
+
if (!columns.some((column) => column.name === "source_options")) {
|
|
166
|
+
db.exec("ALTER TABLE knowledge_bases ADD COLUMN source_options TEXT");
|
|
167
|
+
}
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
if (migrations[v])
|
|
171
|
+
db.exec(migrations[v]);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
// --- CRUD: Knowledge Bases ---
|
|
175
|
+
export function createKB(db, opts) {
|
|
176
|
+
const id = randomUUID();
|
|
177
|
+
const now = Date.now();
|
|
178
|
+
db.prepare(`INSERT INTO knowledge_bases (id, name, description, source_path, source_type, source_options, created_at, updated_at)
|
|
179
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(id, opts.name, opts.description ?? null, opts.source_path ?? null, opts.source_type, opts.source_options ?? null, now, now);
|
|
180
|
+
const kb = getKB(db, id);
|
|
181
|
+
if (!kb)
|
|
182
|
+
throw new Error(`Failed to create knowledge base: ${id}`);
|
|
183
|
+
return kb;
|
|
184
|
+
}
|
|
185
|
+
export function getKB(db, id) {
|
|
186
|
+
return db.prepare("SELECT * FROM knowledge_bases WHERE id = ?").get(id);
|
|
187
|
+
}
|
|
188
|
+
export function getKBByName(db, name) {
|
|
189
|
+
return db.prepare("SELECT * FROM knowledge_bases WHERE name = ?").get(name);
|
|
190
|
+
}
|
|
191
|
+
export function listKBs(db) {
|
|
192
|
+
return db.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC").all();
|
|
193
|
+
}
|
|
194
|
+
export function deleteKB(db, id) {
|
|
195
|
+
db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
|
|
196
|
+
db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
|
|
197
|
+
db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
|
|
198
|
+
}
|
|
199
|
+
export function updateKBStatus(db, id, status) {
|
|
200
|
+
db.prepare("UPDATE knowledge_bases SET status = ?, updated_at = ? WHERE id = ?").run(status, Date.now(), id);
|
|
201
|
+
}
|
|
202
|
+
export function updateKBCounts(db, id, chunkCount, fileCount) {
|
|
203
|
+
db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(chunkCount, fileCount, Date.now(), id);
|
|
204
|
+
}
|
|
205
|
+
// --- Indexing Jobs ---
|
|
206
|
+
export function startIndexingJob(db, kbId, operation, message) {
|
|
207
|
+
const now = Date.now();
|
|
208
|
+
db.prepare(`INSERT INTO indexing_jobs (
|
|
209
|
+
kb_id, operation, status, phase, message, started_at, updated_at, last_progress_at
|
|
210
|
+
) VALUES (?, ?, 'running', 'starting', ?, ?, ?, ?)
|
|
211
|
+
ON CONFLICT(kb_id) DO UPDATE SET
|
|
212
|
+
operation = excluded.operation,
|
|
213
|
+
status = 'running',
|
|
214
|
+
phase = 'starting',
|
|
215
|
+
message = excluded.message,
|
|
216
|
+
started_at = excluded.started_at,
|
|
217
|
+
updated_at = excluded.updated_at,
|
|
218
|
+
last_progress_at = excluded.last_progress_at,
|
|
219
|
+
processed_files = 0,
|
|
220
|
+
processed_chunks = 0,
|
|
221
|
+
total_files = NULL,
|
|
222
|
+
skipped_total = 0,
|
|
223
|
+
added_chunks = 0,
|
|
224
|
+
removed_chunks = 0,
|
|
225
|
+
unchanged_chunks = 0,
|
|
226
|
+
error_message = NULL`).run(kbId, operation, message, now, now, now);
|
|
227
|
+
}
|
|
228
|
+
export function updateIndexingJob(db, kbId, progress) {
|
|
229
|
+
const current = getIndexingJob(db, kbId);
|
|
230
|
+
if (!current)
|
|
231
|
+
return;
|
|
232
|
+
const now = Date.now();
|
|
233
|
+
db.prepare(`UPDATE indexing_jobs SET
|
|
234
|
+
phase = ?,
|
|
235
|
+
message = ?,
|
|
236
|
+
updated_at = ?,
|
|
237
|
+
last_progress_at = ?,
|
|
238
|
+
processed_files = ?,
|
|
239
|
+
processed_chunks = ?,
|
|
240
|
+
total_files = ?,
|
|
241
|
+
skipped_total = ?,
|
|
242
|
+
added_chunks = ?,
|
|
243
|
+
removed_chunks = ?,
|
|
244
|
+
unchanged_chunks = ?
|
|
245
|
+
WHERE kb_id = ?`).run(progress.phase ?? current.phase, progress.message ?? current.message, now, now, progress.processed_files ?? current.processed_files, progress.processed_chunks ?? current.processed_chunks, progress.total_files === undefined ? current.total_files : progress.total_files, progress.skipped_total ?? current.skipped_total, progress.added_chunks ?? current.added_chunks, progress.removed_chunks ?? current.removed_chunks, progress.unchanged_chunks ?? current.unchanged_chunks, kbId);
|
|
246
|
+
}
|
|
247
|
+
export function finishIndexingJob(db, kbId, status, message, errorMessage) {
|
|
248
|
+
const now = Date.now();
|
|
249
|
+
db.prepare(`UPDATE indexing_jobs SET
|
|
250
|
+
status = ?,
|
|
251
|
+
phase = ?,
|
|
252
|
+
message = ?,
|
|
253
|
+
updated_at = ?,
|
|
254
|
+
last_progress_at = ?,
|
|
255
|
+
error_message = ?
|
|
256
|
+
WHERE kb_id = ?`).run(status, status, message, now, now, errorMessage ?? null, kbId);
|
|
257
|
+
}
|
|
258
|
+
export function getIndexingJob(db, kbId) {
|
|
259
|
+
return db.prepare("SELECT * FROM indexing_jobs WHERE kb_id = ?").get(kbId);
|
|
260
|
+
}
|
|
261
|
+
// --- CRUD: Chunks ---
|
|
262
|
+
export function insertChunks(db, kbId, chunks) {
|
|
263
|
+
const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
|
|
264
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
265
|
+
const now = Date.now();
|
|
266
|
+
const insertMany = db.transaction((items) => {
|
|
267
|
+
for (const c of items) {
|
|
268
|
+
stmt.run(randomUUID(), kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
insertMany(chunks);
|
|
272
|
+
}
|
|
273
|
+
export function getChunksByKB(db, kbId) {
|
|
274
|
+
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
275
|
+
}
|
|
276
|
+
export function iterateChunksByKB(db, kbId) {
|
|
277
|
+
return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId);
|
|
278
|
+
}
|
|
279
|
+
export function getChunkIdsByKB(db, kbId) {
|
|
280
|
+
const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
281
|
+
return rows.map((r) => r.id);
|
|
282
|
+
}
|
|
283
|
+
export function iterateChunkIdsByKB(db, kbId) {
|
|
284
|
+
return db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId);
|
|
285
|
+
}
|
|
286
|
+
export function getChunkById(db, id) {
|
|
287
|
+
return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id);
|
|
288
|
+
}
|
|
289
|
+
export function getChunksByFile(db, kbId, filePath, startLine, endLine) {
|
|
290
|
+
return db
|
|
291
|
+
.prepare(`SELECT * FROM chunks
|
|
292
|
+
WHERE kb_id = ? AND file_path = ? AND end_line >= ? AND start_line <= ?
|
|
293
|
+
ORDER BY start_line, end_line`)
|
|
294
|
+
.all(kbId, filePath, startLine, endLine);
|
|
295
|
+
}
|
|
296
|
+
export function getChunkByRowid(db, rowid) {
|
|
297
|
+
return db.prepare("SELECT * FROM chunks WHERE rowid = ?").get(rowid);
|
|
298
|
+
}
|
|
299
|
+
export function deleteChunksByKB(db, kbId) {
|
|
300
|
+
db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(kbId);
|
|
301
|
+
}
|
|
302
|
+
export function deleteChunksByIds(db, ids) {
|
|
303
|
+
if (ids.length === 0)
|
|
304
|
+
return;
|
|
305
|
+
const batchSize = 500;
|
|
306
|
+
const deleteBatch = db.transaction((batch) => {
|
|
307
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
308
|
+
db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...batch);
|
|
309
|
+
});
|
|
310
|
+
for (let offset = 0; offset < ids.length; offset += batchSize) {
|
|
311
|
+
deleteBatch(ids.slice(offset, offset + batchSize));
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
export function getChunkHashesByKB(db, kbId) {
|
|
315
|
+
const rows = db.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
|
|
316
|
+
return new Map(rows.map((r) => [r.content_hash, r.id]));
|
|
317
|
+
}
|
|
318
|
+
export function iterateChunkHashesByKB(db, kbId) {
|
|
319
|
+
return db
|
|
320
|
+
.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid")
|
|
321
|
+
.iterate(kbId);
|
|
322
|
+
}
|
|
323
|
+
export function getChunkCount(db, kbId) {
|
|
324
|
+
const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
|
325
|
+
return row.count;
|
|
326
|
+
}
|
|
327
|
+
export function getFileCount(db, kbId) {
|
|
328
|
+
const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
|
|
329
|
+
return row.count;
|
|
330
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void;
|
|
2
|
+
export declare function stopWatcher(kbId: string): void;
|
|
3
|
+
export declare function stopAllWatchers(): void;
|
|
4
|
+
export declare function getActiveWatcherCount(): number;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { existsSync, readdirSync, statSync, watch } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
const watchers = new Map();
|
|
4
|
+
const pollers = new Map();
|
|
5
|
+
const snapshots = new Map();
|
|
6
|
+
const debounceTimers = new Map();
|
|
7
|
+
const DEBOUNCE_MS = 2000;
|
|
8
|
+
const POLL_MS = 2000;
|
|
9
|
+
function scheduleUpdate(kbId, onUpdate) {
|
|
10
|
+
const existing = debounceTimers.get(kbId);
|
|
11
|
+
if (existing)
|
|
12
|
+
clearTimeout(existing);
|
|
13
|
+
debounceTimers.set(kbId, setTimeout(() => {
|
|
14
|
+
debounceTimers.delete(kbId);
|
|
15
|
+
onUpdate(kbId);
|
|
16
|
+
}, DEBOUNCE_MS));
|
|
17
|
+
}
|
|
18
|
+
function scanSnapshot(dirPath) {
|
|
19
|
+
const snapshot = new Map();
|
|
20
|
+
function scan(dir) {
|
|
21
|
+
let entries;
|
|
22
|
+
try {
|
|
23
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
for (const entry of entries) {
|
|
29
|
+
const fullPath = join(dir, entry.name);
|
|
30
|
+
try {
|
|
31
|
+
const stat = statSync(fullPath);
|
|
32
|
+
if (entry.isDirectory()) {
|
|
33
|
+
scan(fullPath);
|
|
34
|
+
}
|
|
35
|
+
else if (entry.isFile()) {
|
|
36
|
+
snapshot.set(fullPath, `${stat.mtimeMs}:${stat.size}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
/* file disappeared or is unreadable */
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (existsSync(dirPath))
|
|
45
|
+
scan(dirPath);
|
|
46
|
+
return snapshot;
|
|
47
|
+
}
|
|
48
|
+
function snapshotsDiffer(a, b) {
|
|
49
|
+
if (a.size !== b.size)
|
|
50
|
+
return true;
|
|
51
|
+
for (const [path, value] of a) {
|
|
52
|
+
if (b.get(path) !== value)
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
function startPoller(kbId, dirPath, onUpdate) {
|
|
58
|
+
snapshots.set(kbId, scanSnapshot(dirPath));
|
|
59
|
+
pollers.set(kbId, setInterval(() => {
|
|
60
|
+
const previous = snapshots.get(kbId) ?? new Map();
|
|
61
|
+
const next = scanSnapshot(dirPath);
|
|
62
|
+
if (snapshotsDiffer(previous, next)) {
|
|
63
|
+
snapshots.set(kbId, next);
|
|
64
|
+
scheduleUpdate(kbId, onUpdate);
|
|
65
|
+
}
|
|
66
|
+
}, POLL_MS));
|
|
67
|
+
}
|
|
68
|
+
export function startWatcher(kbId, dirPath, onUpdate) {
|
|
69
|
+
stopWatcher(kbId);
|
|
70
|
+
startPoller(kbId, dirPath, onUpdate);
|
|
71
|
+
try {
|
|
72
|
+
const watcher = watch(dirPath, { recursive: true }, () => {
|
|
73
|
+
snapshots.set(kbId, scanSnapshot(dirPath));
|
|
74
|
+
scheduleUpdate(kbId, onUpdate);
|
|
75
|
+
});
|
|
76
|
+
watcher.on("error", () => {
|
|
77
|
+
watchers.get(kbId)?.close();
|
|
78
|
+
watchers.delete(kbId);
|
|
79
|
+
});
|
|
80
|
+
watchers.set(kbId, watcher);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
/* polling fallback remains active */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export function stopWatcher(kbId) {
|
|
87
|
+
watchers.get(kbId)?.close();
|
|
88
|
+
watchers.delete(kbId);
|
|
89
|
+
const poller = pollers.get(kbId);
|
|
90
|
+
if (poller) {
|
|
91
|
+
clearInterval(poller);
|
|
92
|
+
pollers.delete(kbId);
|
|
93
|
+
}
|
|
94
|
+
snapshots.delete(kbId);
|
|
95
|
+
const t = debounceTimers.get(kbId);
|
|
96
|
+
if (t) {
|
|
97
|
+
clearTimeout(t);
|
|
98
|
+
debounceTimers.delete(kbId);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
export function stopAllWatchers() {
|
|
102
|
+
const ids = new Set([...watchers.keys(), ...pollers.keys()]);
|
|
103
|
+
for (const id of ids)
|
|
104
|
+
stopWatcher(id);
|
|
105
|
+
}
|
|
106
|
+
export function getActiveWatcherCount() {
|
|
107
|
+
return new Set([...watchers.keys(), ...pollers.keys()]).size;
|
|
108
|
+
}
|
package/extension.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
|
|
4
|
+
const distUrl = new URL("./dist/index.js", import.meta.url);
|
|
5
|
+
const sourceUrl = new URL("./index.ts", import.meta.url);
|
|
6
|
+
const entryUrl = existsSync(fileURLToPath(distUrl)) ? distUrl : sourceUrl;
|
|
7
|
+
const module = await import(entryUrl.href);
|
|
8
|
+
|
|
9
|
+
export default module.default;
|
package/package.json
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.3",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
|
-
"main": "index.
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
7
8
|
"pi": {
|
|
8
9
|
"extensions": [
|
|
9
|
-
"./
|
|
10
|
+
"./extension.js"
|
|
10
11
|
]
|
|
11
12
|
},
|
|
12
13
|
"files": [
|
|
13
|
-
"
|
|
14
|
-
"
|
|
14
|
+
"extension.js",
|
|
15
|
+
"dist/",
|
|
15
16
|
".pi/",
|
|
16
17
|
"README.md",
|
|
17
18
|
"CHANGELOG.md",
|
|
@@ -36,9 +37,12 @@
|
|
|
36
37
|
"embeddings"
|
|
37
38
|
],
|
|
38
39
|
"scripts": {
|
|
40
|
+
"build": "tsc -p tsconfig.build.json",
|
|
39
41
|
"check": "biome check --write --error-on-warnings .",
|
|
42
|
+
"prepack": "npm run typecheck && npm run build",
|
|
40
43
|
"test": "vitest --run test/unit/",
|
|
41
44
|
"test:e2e": "vitest --run test/e2e/",
|
|
45
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
42
46
|
"bench": "vitest bench test/bench/"
|
|
43
47
|
},
|
|
44
48
|
"dependencies": {
|
|
@@ -58,6 +62,7 @@
|
|
|
58
62
|
"@biomejs/biome": "2.3.5",
|
|
59
63
|
"@types/better-sqlite3": "7.6.13",
|
|
60
64
|
"@types/node": "22.19.19",
|
|
65
|
+
"typescript": "5.9.3",
|
|
61
66
|
"vitest": "3.2.4"
|
|
62
67
|
},
|
|
63
68
|
"engines": {
|