pi-knowledge 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,31 @@
1
+ import type Database from "better-sqlite3";
2
+ import { preTokenizeForFTS } from "../indexer/chunker.ts";
3
+
4
+ export interface BM25Result { chunkId: string; score: number; }
5
+
6
+ function prepareFtsQuery(query: string): string {
7
+ let q = preTokenizeForFTS(query);
8
+ q = q.replace(/[*"(){}[\]^~:+.#@!\\/<>|&$%]/g, " ");
9
+ const terms = q.split(/\s+/).filter((t) => t.length > 0);
10
+ if (terms.length === 0) return "";
11
+ return terms.join(" AND ");
12
+ }
13
+
14
+ export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
15
+ const ftsQuery = prepareFtsQuery(query);
16
+ if (!ftsQuery) return [];
17
+ try {
18
+ if (kbId) {
19
+ return db.prepare(
20
+ `SELECT c.id as chunkId, bm25(chunks_fts) as score
21
+ FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
22
+ WHERE chunks_fts MATCH ? AND c.kb_id = ? ORDER BY score LIMIT ?`,
23
+ ).all(ftsQuery, kbId, limit) as BM25Result[];
24
+ }
25
+ return db.prepare(
26
+ `SELECT c.id as chunkId, bm25(chunks_fts) as score
27
+ FROM chunks_fts JOIN chunks c ON chunks_fts.rowid = c.rowid
28
+ WHERE chunks_fts MATCH ? ORDER BY score LIMIT ?`,
29
+ ).all(ftsQuery, limit) as BM25Result[];
30
+ } catch { return []; }
31
+ }
@@ -0,0 +1,14 @@
1
+ export interface FusionResult { chunkId: string; score: number; }
2
+
3
+ export function reciprocalRankFusion(lists: Array<{ chunkId: string; score: number }[]>, k = 60): FusionResult[] {
4
+ const scores = new Map<string, number>();
5
+ for (const list of lists) {
6
+ for (let rank = 0; rank < list.length; rank++) {
7
+ const { chunkId } = list[rank];
8
+ scores.set(chunkId, (scores.get(chunkId) ?? 0) + 1 / (k + rank + 1));
9
+ }
10
+ }
11
+ return [...scores.entries()]
12
+ .map(([chunkId, score]) => ({ chunkId, score }))
13
+ .sort((a, b) => b.score - a.score);
14
+ }
@@ -0,0 +1,42 @@
1
+ import { join } from "node:path";
2
+ import { getDefaultKnowledgeDir } from "../storage/sqlite.ts";
3
+
4
+ let rerankerPipeline: any = null;
5
+ let disposeTimer: ReturnType<typeof setTimeout> | null = null;
6
+ const IDLE_MS = 60_000;
7
+
8
+ async function load(): Promise<any> {
9
+ if (rerankerPipeline) return rerankerPipeline;
10
+ const { pipeline, env } = await import("@huggingface/transformers");
11
+ env.cacheDir = join(getDefaultKnowledgeDir(), "models");
12
+ rerankerPipeline = await pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-4-v2");
13
+ return rerankerPipeline;
14
+ }
15
+
16
+ function resetTimer(): void {
17
+ if (disposeTimer) clearTimeout(disposeTimer);
18
+ disposeTimer = setTimeout(() => disposeReranker(), IDLE_MS);
19
+ }
20
+
21
+ export async function disposeReranker(): Promise<void> {
22
+ if (disposeTimer) { clearTimeout(disposeTimer); disposeTimer = null; }
23
+ if (rerankerPipeline) { await rerankerPipeline.dispose(); rerankerPipeline = null; }
24
+ }
25
+
26
+ export interface RerankCandidate { chunkId: string; content: string; }
27
+
28
+ export async function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{ chunkId: string; score: number }>> {
29
+ if (candidates.length === 0) return [];
30
+ const pipe = await load();
31
+ resetTimer();
32
+
33
+ const results: Array<{ chunkId: string; score: number }> = [];
34
+ for (const c of candidates) {
35
+ const output = await pipe({ text: query, text_pair: c.content });
36
+ const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
37
+ results.push({ chunkId: c.chunkId, score });
38
+ }
39
+
40
+ results.sort((a, b) => b.score - a.score);
41
+ return results.slice(0, topK);
42
+ }
@@ -0,0 +1,13 @@
1
+ export interface VectorResult { chunkId: string; score: number; }
2
+
3
+ function cosine(a: Float32Array, b: Float32Array): number {
4
+ let dot = 0;
5
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
6
+ return dot;
7
+ }
8
+
9
+ export function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit = 50): VectorResult[] {
10
+ const scores = vectors.map((v, i) => ({ chunkId: chunkIds[i], score: cosine(queryVec, v) }));
11
+ scores.sort((a, b) => b.score - a.score);
12
+ return scores.slice(0, limit);
13
+ }
@@ -0,0 +1,222 @@
1
+ import Database from "better-sqlite3";
2
+ import { existsSync, mkdirSync } from "node:fs";
3
+ import { join } from "node:path";
4
+ import { homedir } from "node:os";
5
+ import { randomUUID } from "node:crypto";
6
+
7
+ export interface KnowledgeBase {
8
+ id: string;
9
+ name: string;
10
+ description: string | null;
11
+ source_path: string | null;
12
+ source_type: "file" | "directory" | "text";
13
+ created_at: number;
14
+ updated_at: number;
15
+ chunk_count: number;
16
+ file_count: number;
17
+ embedding_model: string;
18
+ status: "ready" | "indexing" | "error" | "stale";
19
+ }
20
+
21
+ export interface Chunk {
22
+ id: string;
23
+ kb_id: string;
24
+ content_hash: string;
25
+ content: string;
26
+ content_tokenized: string;
27
+ file_path: string;
28
+ file_type: string;
29
+ start_line: number;
30
+ end_line: number;
31
+ metadata_json: string;
32
+ indexed_at: number;
33
+ }
34
+
35
+ export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
36
+
37
+ const SCHEMA_VERSION = 1;
38
+
39
+ const SCHEMA_SQL = `
40
+ CREATE TABLE IF NOT EXISTS schema_version (
41
+ version INTEGER NOT NULL
42
+ );
43
+
44
+ CREATE TABLE IF NOT EXISTS knowledge_bases (
45
+ id TEXT PRIMARY KEY,
46
+ name TEXT NOT NULL,
47
+ description TEXT,
48
+ source_path TEXT,
49
+ source_type TEXT NOT NULL DEFAULT 'directory',
50
+ created_at INTEGER NOT NULL,
51
+ updated_at INTEGER NOT NULL,
52
+ chunk_count INTEGER NOT NULL DEFAULT 0,
53
+ file_count INTEGER NOT NULL DEFAULT 0,
54
+ embedding_model TEXT NOT NULL DEFAULT 'multilingual-e5-small',
55
+ status TEXT NOT NULL DEFAULT 'ready'
56
+ );
57
+
58
+ CREATE TABLE IF NOT EXISTS chunks (
59
+ id TEXT PRIMARY KEY,
60
+ kb_id TEXT NOT NULL,
61
+ content_hash TEXT NOT NULL,
62
+ content TEXT NOT NULL,
63
+ content_tokenized TEXT NOT NULL,
64
+ file_path TEXT NOT NULL,
65
+ file_type TEXT NOT NULL DEFAULT 'text',
66
+ start_line INTEGER NOT NULL DEFAULT 0,
67
+ end_line INTEGER NOT NULL DEFAULT 0,
68
+ metadata_json TEXT NOT NULL DEFAULT '{}',
69
+ indexed_at INTEGER NOT NULL,
70
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
71
+ );
72
+
73
+ CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
74
+ CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
75
+
76
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
77
+ content_tokenized,
78
+ content=chunks,
79
+ content_rowid=rowid
80
+ );
81
+
82
+ -- Triggers to keep FTS in sync with chunks table
83
+ CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
84
+ INSERT INTO chunks_fts(rowid, content_tokenized) VALUES (new.rowid, new.content_tokenized);
85
+ END;
86
+
87
+ CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
88
+ INSERT INTO chunks_fts(chunks_fts, rowid, content_tokenized) VALUES('delete', old.rowid, old.content_tokenized);
89
+ END;
90
+
91
+ CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
92
+ INSERT INTO chunks_fts(chunks_fts, rowid, content_tokenized) VALUES('delete', old.rowid, old.content_tokenized);
93
+ INSERT INTO chunks_fts(rowid, content_tokenized) VALUES (new.rowid, new.content_tokenized);
94
+ END;
95
+ `;
96
+
97
+ export function getDefaultKnowledgeDir(): string {
98
+ return join(homedir(), ".pi", "knowledge");
99
+ }
100
+
101
+ export function openDatabase(knowledgeDir?: string): Database.Database {
102
+ const dir = knowledgeDir ?? getDefaultKnowledgeDir();
103
+ if (!existsSync(dir)) {
104
+ mkdirSync(dir, { recursive: true });
105
+ }
106
+ const dbPath = join(dir, "knowledge.db");
107
+ const db = new Database(dbPath);
108
+ db.pragma("journal_mode = WAL");
109
+ db.pragma("busy_timeout = 5000");
110
+ db.pragma("foreign_keys = ON");
111
+
112
+ const hasVersion = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'").get();
113
+ if (!hasVersion) {
114
+ db.exec(SCHEMA_SQL);
115
+ db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
116
+ }
117
+
118
+ return db;
119
+ }
120
+
121
+ // --- CRUD: Knowledge Bases ---
122
+
123
+ export function createKB(
124
+ db: Database.Database,
125
+ opts: { name: string; description?: string; source_path?: string; source_type: KnowledgeBase["source_type"] },
126
+ ): KnowledgeBase {
127
+ const id = randomUUID();
128
+ const now = Date.now();
129
+ db.prepare(
130
+ `INSERT INTO knowledge_bases (id, name, description, source_path, source_type, created_at, updated_at)
131
+ VALUES (?, ?, ?, ?, ?, ?, ?)`,
132
+ ).run(id, opts.name, opts.description ?? null, opts.source_path ?? null, opts.source_type, now, now);
133
+ return getKB(db, id)!;
134
+ }
135
+
136
+ export function getKB(db: Database.Database, id: string): KnowledgeBase | undefined {
137
+ return db.prepare("SELECT * FROM knowledge_bases WHERE id = ?").get(id) as KnowledgeBase | undefined;
138
+ }
139
+
140
+ export function getKBByName(db: Database.Database, name: string): KnowledgeBase | undefined {
141
+ return db.prepare("SELECT * FROM knowledge_bases WHERE name = ?").get(name) as KnowledgeBase | undefined;
142
+ }
143
+
144
+ export function listKBs(db: Database.Database): KnowledgeBase[] {
145
+ return db.prepare("SELECT * FROM knowledge_bases ORDER BY updated_at DESC").all() as KnowledgeBase[];
146
+ }
147
+
148
+ export function deleteKB(db: Database.Database, id: string): void {
149
+ db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
150
+ db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
151
+ }
152
+
153
+ export function updateKBStatus(db: Database.Database, id: string, status: KnowledgeBase["status"]): void {
154
+ db.prepare("UPDATE knowledge_bases SET status = ?, updated_at = ? WHERE id = ?").run(status, Date.now(), id);
155
+ }
156
+
157
+ export function updateKBCounts(db: Database.Database, id: string, chunkCount: number, fileCount: number): void {
158
+ db.prepare("UPDATE knowledge_bases SET chunk_count = ?, file_count = ?, updated_at = ? WHERE id = ?").run(
159
+ chunkCount,
160
+ fileCount,
161
+ Date.now(),
162
+ id,
163
+ );
164
+ }
165
+
166
+ // --- CRUD: Chunks ---
167
+
168
+ export function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void {
169
+ const stmt = db.prepare(
170
+ `INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
171
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
172
+ );
173
+ const now = Date.now();
174
+ const insertMany = db.transaction((items: ChunkInsert[]) => {
175
+ for (const c of items) {
176
+ stmt.run(randomUUID(), kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
177
+ }
178
+ });
179
+ insertMany(chunks);
180
+ }
181
+
182
+ export function getChunksByKB(db: Database.Database, kbId: string): Chunk[] {
183
+ return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as Chunk[];
184
+ }
185
+
186
+ export function getChunkIdsByKB(db: Database.Database, kbId: string): string[] {
187
+ const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as { id: string }[];
188
+ return rows.map((r) => r.id);
189
+ }
190
+
191
+ export function getChunkById(db: Database.Database, id: string): Chunk | undefined {
192
+ return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id) as Chunk | undefined;
193
+ }
194
+
195
+ export function getChunkByRowid(db: Database.Database, rowid: number): Chunk | undefined {
196
+ return db.prepare("SELECT * FROM chunks WHERE rowid = ?").get(rowid) as Chunk | undefined;
197
+ }
198
+
199
+ export function deleteChunksByKB(db: Database.Database, kbId: string): void {
200
+ db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(kbId);
201
+ }
202
+
203
+ export function deleteChunksByIds(db: Database.Database, ids: string[]): void {
204
+ if (ids.length === 0) return;
205
+ const placeholders = ids.map(() => "?").join(",");
206
+ db.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...ids);
207
+ }
208
+
209
+ export function getChunkHashesByKB(db: Database.Database, kbId: string): Map<string, string> {
210
+ const rows = db.prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId) as { id: string; content_hash: string }[];
211
+ return new Map(rows.map((r) => [r.content_hash, r.id]));
212
+ }
213
+
214
+ export function getChunkCount(db: Database.Database, kbId: string): number {
215
+ const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId) as { count: number };
216
+ return row.count;
217
+ }
218
+
219
+ export function getFileCount(db: Database.Database, kbId: string): number {
220
+ const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId) as { count: number };
221
+ return row.count;
222
+ }
@@ -0,0 +1,36 @@
1
+ import { watch, type FSWatcher } from "node:fs";
2
+
3
+ const watchers = new Map<string, FSWatcher>();
4
+ const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
5
+ const DEBOUNCE_MS = 2000;
6
+
7
+ export function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void {
8
+ stopWatcher(kbId);
9
+ try {
10
+ const watcher = watch(dirPath, { recursive: true }, () => {
11
+ const existing = debounceTimers.get(kbId);
12
+ if (existing) clearTimeout(existing);
13
+ debounceTimers.set(kbId, setTimeout(() => {
14
+ debounceTimers.delete(kbId);
15
+ onUpdate(kbId);
16
+ }, DEBOUNCE_MS));
17
+ });
18
+ watcher.on("error", () => stopWatcher(kbId));
19
+ watchers.set(kbId, watcher);
20
+ } catch { /* fail silently if recursive watch not supported */ }
21
+ }
22
+
23
+ export function stopWatcher(kbId: string): void {
24
+ watchers.get(kbId)?.close();
25
+ watchers.delete(kbId);
26
+ const t = debounceTimers.get(kbId);
27
+ if (t) { clearTimeout(t); debounceTimers.delete(kbId); }
28
+ }
29
+
30
+ export function stopAllWatchers(): void {
31
+ for (const [id] of watchers) stopWatcher(id);
32
+ }
33
+
34
+ export function getActiveWatcherCount(): number {
35
+ return watchers.size;
36
+ }