pi-knowledge 0.4.7 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ import type { KnowledgeSymbolInsert } from "../storage/sqlite.ts";
2
+ export declare function extractSymbols(content: string, filePath: string, fileType: string): KnowledgeSymbolInsert[];
@@ -0,0 +1,175 @@
1
+ import { extname } from "node:path";
2
+ const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java"]);
3
+ const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
4
+ function lineNumberAt(content, offset) {
5
+ let line = 1;
6
+ for (let i = 0; i < offset; i++) {
7
+ if (content.charCodeAt(i) === 10)
8
+ line++;
9
+ }
10
+ return line;
11
+ }
12
+ function lineEnd(content, startLine, maxLines = 12) {
13
+ return Math.min(startLine + maxLines, content.split("\n").length);
14
+ }
15
+ function makeSymbol(name, kind, filePath, fileType, startLine, options = {}) {
16
+ const trimmed = name.trim();
17
+ if (!trimmed)
18
+ return undefined;
19
+ return {
20
+ name: trimmed,
21
+ kind,
22
+ file_path: filePath,
23
+ file_type: fileType,
24
+ start_line: startLine,
25
+ end_line: options.endLine ?? startLine,
26
+ container_name: options.containerName ?? null,
27
+ signature: options.signature ?? null,
28
+ text: options.text ?? trimmed,
29
+ metadata_json: JSON.stringify(options.metadata ?? {}),
30
+ };
31
+ }
32
+ function pushSymbol(symbols, symbol) {
33
+ if (symbol)
34
+ symbols.push(symbol);
35
+ }
36
+ function extractMarkdownSymbols(content, filePath, fileType) {
37
+ const symbols = [];
38
+ const lines = content.split("\n");
39
+ for (let index = 0; index < lines.length; index++) {
40
+ const match = /^(#{1,6})\s+(.+?)\s*$/.exec(lines[index]);
41
+ if (!match)
42
+ continue;
43
+ pushSymbol(symbols, makeSymbol(match[2].replace(/\s+#*$/, ""), "heading", filePath, fileType, index + 1, {
44
+ text: lines[index],
45
+ metadata: { depth: match[1].length },
46
+ }));
47
+ }
48
+ return symbols;
49
+ }
50
+ function extractConfigSymbols(content, filePath, fileType) {
51
+ const symbols = [];
52
+ const lines = content.split("\n");
53
+ const ext = extname(filePath).toLowerCase();
54
+ for (let index = 0; index < lines.length; index++) {
55
+ const line = lines[index];
56
+ if (ext === ".json") {
57
+ const match = /^\s*"([^"]+)"\s*:/.exec(line);
58
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
59
+ continue;
60
+ }
61
+ if (ext === ".yaml" || ext === ".yml") {
62
+ const match = /^\s*([A-Za-z0-9_.-]+)\s*:/.exec(line);
63
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
64
+ continue;
65
+ }
66
+ if (ext === ".toml") {
67
+ const section = /^\s*\[+([A-Za-z0-9_.-]+)\]+/.exec(line);
68
+ const key = /^\s*([A-Za-z0-9_.-]+)\s*=/.exec(line);
69
+ const name = section?.[1] ?? key?.[1];
70
+ pushSymbol(symbols, name ? makeSymbol(name, "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
71
+ continue;
72
+ }
73
+ const env = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
74
+ pushSymbol(symbols, env ? makeSymbol(env[1], "env_var", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
75
+ }
76
+ return symbols;
77
+ }
78
+ function extractCodeSymbols(content, filePath, fileType) {
79
+ const symbols = [];
80
+ const patterns = [
81
+ {
82
+ regex: /\b(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)/g,
83
+ kind: "function",
84
+ nameIndex: 1,
85
+ },
86
+ { regex: /\b(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b/g, kind: "class", nameIndex: 1 },
87
+ { regex: /\b(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/g, kind: "interface", nameIndex: 1 },
88
+ { regex: /\b(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/g, kind: "type", nameIndex: 1 },
89
+ {
90
+ regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g,
91
+ kind: "function",
92
+ nameIndex: 1,
93
+ },
94
+ { regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/g, kind: "variable", nameIndex: 1 },
95
+ { regex: /\bfunc\s+(?:\([^)]+\)\s*)?([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
96
+ { regex: /\btype\s+([A-Za-z_][\w]*)\s+(?:struct|interface)\b/g, kind: "type", nameIndex: 1 },
97
+ { regex: /\bdef\s+([A-Za-z_][\w]*)\s*\([^)]*\)\s*:/g, kind: "function", nameIndex: 1 },
98
+ { regex: /\bclass\s+([A-Za-z_][\w]*)\b/g, kind: "class", nameIndex: 1 },
99
+ { regex: /\bfn\s+([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
100
+ { regex: /\b(?:struct|enum|trait)\s+([A-Za-z_][\w]*)\b/g, kind: "type", nameIndex: 1 },
101
+ ];
102
+ const seen = new Set();
103
+ for (const pattern of patterns) {
104
+ for (const match of content.matchAll(pattern.regex)) {
105
+ const name = match[pattern.nameIndex];
106
+ const index = match.index ?? 0;
107
+ const startLine = lineNumberAt(content, index);
108
+ const key = `${pattern.kind}:${name}:${startLine}`;
109
+ if (seen.has(key))
110
+ continue;
111
+ seen.add(key);
112
+ pushSymbol(symbols, makeSymbol(name, pattern.kind, filePath, fileType, startLine, {
113
+ endLine: lineEnd(content, startLine),
114
+ signature: match[0].split("\n")[0].trim(),
115
+ text: match[0].split("\n")[0].trim(),
116
+ }));
117
+ }
118
+ }
119
+ return symbols;
120
+ }
121
+ function extractRouteSymbols(content, filePath, fileType) {
122
+ const symbols = [];
123
+ const patterns = [
124
+ {
125
+ regex: /\b(?:app|router|route|server|api|hono)\s*\.\s*(get|post|put|patch|delete|del|options|head|all|use)\s*\(\s*["'`]([^"'`]+)["'`]/g,
126
+ methodIndex: 1,
127
+ pathIndex: 2,
128
+ },
129
+ {
130
+ regex: /\b(?:http\.)?HandleFunc\s*\(\s*["'`]([^"'`]+)["'`]/g,
131
+ pathIndex: 1,
132
+ defaultMethod: "ANY",
133
+ },
134
+ {
135
+ regex: /\b(?:http\.)?Handle\s*\(\s*["'`]([^"'`]+)["'`]/g,
136
+ pathIndex: 1,
137
+ defaultMethod: "ANY",
138
+ },
139
+ ];
140
+ const seen = new Set();
141
+ for (const pattern of patterns) {
142
+ for (const match of content.matchAll(pattern.regex)) {
143
+ const rawMethod = pattern.methodIndex ? match[pattern.methodIndex] : pattern.defaultMethod;
144
+ const method = rawMethod === "del" ? "DELETE" : (rawMethod ?? "ANY").toUpperCase();
145
+ const path = match[pattern.pathIndex];
146
+ const index = match.index ?? 0;
147
+ const startLine = lineNumberAt(content, index);
148
+ const name = `${method} ${path}`;
149
+ const key = `${name}:${startLine}`;
150
+ if (seen.has(key))
151
+ continue;
152
+ seen.add(key);
153
+ pushSymbol(symbols, makeSymbol(name, "route", filePath, fileType, startLine, {
154
+ signature: match[0].split("\n")[0].trim(),
155
+ text: match[0].split("\n")[0].trim(),
156
+ metadata: { method, path },
157
+ }));
158
+ }
159
+ }
160
+ return symbols;
161
+ }
162
+ export function extractSymbols(content, filePath, fileType) {
163
+ const ext = extname(filePath).toLowerCase();
164
+ const symbols = [];
165
+ if (fileType === "markdown" || ext === ".md" || ext === ".mdx")
166
+ symbols.push(...extractMarkdownSymbols(content, filePath, fileType));
167
+ if (CONFIG_EXTENSIONS.has(ext) || filePath.split("/").pop()?.startsWith(".env")) {
168
+ symbols.push(...extractConfigSymbols(content, filePath, fileType));
169
+ }
170
+ if (CODE_EXTENSIONS.has(ext) || ["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
171
+ symbols.push(...extractCodeSymbols(content, filePath, fileType));
172
+ symbols.push(...extractRouteSymbols(content, filePath, fileType));
173
+ }
174
+ return symbols;
175
+ }
@@ -3,7 +3,7 @@ export interface RerankWorkerCandidate {
3
3
  chunkId: string;
4
4
  content: string;
5
5
  }
6
- export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
6
+ export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
7
7
  chunkId: string;
8
8
  score: number;
9
9
  }>>;
@@ -86,19 +86,19 @@ async function requestModelWorker(message, signal) {
86
86
  const id = nextRequestId++;
87
87
  return new Promise((resolve, reject) => {
88
88
  let abortHandler;
89
+ const cleanup = () => {
90
+ if (abortHandler)
91
+ signal?.removeEventListener("abort", abortHandler);
92
+ };
89
93
  pending.set(id, { resolve, reject });
90
94
  if (signal) {
91
95
  abortHandler = () => {
92
96
  pending.delete(id);
97
+ cleanup();
93
98
  reject(new Error("Cancelled"));
94
- shutdownModelWorker();
95
99
  };
96
100
  signal.addEventListener("abort", abortHandler, { once: true });
97
101
  }
98
- const cleanup = () => {
99
- if (abortHandler)
100
- signal?.removeEventListener("abort", abortHandler);
101
- };
102
102
  const originalResolve = resolve;
103
103
  const originalReject = reject;
104
104
  pending.set(id, {
@@ -130,8 +130,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
130
130
  return new Float32Array(vector);
131
131
  });
132
132
  }
133
- export async function rerankInModelWorker(query, candidates, topK) {
134
- const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
133
+ export async function rerankInModelWorker(query, candidates, topK, signal) {
134
+ const result = await requestModelWorker({ type: "rerank", query, candidates, topK }, signal);
135
135
  if (!Array.isArray(result))
136
136
  throw new Error("Invalid reranker worker response");
137
137
  return result.map((item) => {
@@ -4,7 +4,7 @@ export interface RerankCandidate {
4
4
  chunkId: string;
5
5
  content: string;
6
6
  }
7
- export declare function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{
7
+ export declare function rerank(query: string, candidates: RerankCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
8
8
  chunkId: string;
9
9
  score: number;
10
10
  }>>;
@@ -53,12 +53,12 @@ export async function prepareRerankerForShutdown() {
53
53
  clearTimer();
54
54
  await waitForNoActiveRuns();
55
55
  }
56
- export async function rerank(query, candidates, topK) {
56
+ export async function rerank(query, candidates, topK, signal) {
57
57
  if (candidates.length === 0)
58
58
  return [];
59
59
  beginRun();
60
60
  try {
61
- return await rerankInModelWorker(query, candidates, topK);
61
+ return await rerankInModelWorker(query, candidates, topK, signal);
62
62
  }
63
63
  finally {
64
64
  endRun();
@@ -10,5 +10,5 @@ type ChunkIdSource = Iterable<string | {
10
10
  id: string;
11
11
  }>;
12
12
  export declare function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit?: number): VectorResult[];
13
- export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number): VectorFileSearchResult;
13
+ export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number, signal?: AbortSignal): VectorFileSearchResult;
14
14
  export {};
@@ -25,7 +25,9 @@ export function searchVector(queryVec, vectors, chunkIds, limit = 50) {
25
25
  scores.sort((a, b) => b.score - a.score);
26
26
  return scores.slice(0, limit);
27
27
  }
28
- export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
28
+ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50, signal) {
29
+ if (signal?.aborted)
30
+ throw new Error("Cancelled");
29
31
  const reader = openVectorReader(vectorPath);
30
32
  if (!reader)
31
33
  return { results: [], vectorsByChunkId: new Map() };
@@ -36,6 +38,8 @@ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
36
38
  const top = [];
37
39
  let i = 0;
38
40
  for (const chunkIdRef of chunkIds) {
41
+ if (signal?.aborted)
42
+ throw new Error("Cancelled");
39
43
  if (i >= reader.count)
40
44
  break;
41
45
  const vectorIndex = i;
@@ -45,6 +45,23 @@ export interface Chunk {
45
45
  indexed_at: number;
46
46
  }
47
47
  export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
48
+ export interface KnowledgeSymbol {
49
+ id: string;
50
+ kb_id: string;
51
+ name: string;
52
+ normalized_name: string;
53
+ kind: "function" | "class" | "interface" | "type" | "variable" | "heading" | "config_key" | "env_var" | "route";
54
+ file_path: string;
55
+ file_type: string;
56
+ start_line: number;
57
+ end_line: number;
58
+ container_name: string | null;
59
+ signature: string | null;
60
+ text: string;
61
+ metadata_json: string;
62
+ indexed_at: number;
63
+ }
64
+ export type KnowledgeSymbolInsert = Omit<KnowledgeSymbol, "id" | "kb_id" | "indexed_at" | "normalized_name">;
48
65
  export declare function getDefaultKnowledgeDir(): string;
49
66
  export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
50
67
  legacyPiDir?: string;
@@ -78,7 +95,7 @@ export declare function updateIndexingJob(db: Database.Database, kbId: string, p
78
95
  }): void;
79
96
  export declare function finishIndexingJob(db: Database.Database, kbId: string, status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">, message: string, errorMessage?: string): void;
80
97
  export declare function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined;
81
- export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void;
98
+ export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): string[];
82
99
  export declare function getChunksByKB(db: Database.Database, kbId: string): Chunk[];
83
100
  export declare function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk>;
84
101
  export declare function getChunkIdsByKB(db: Database.Database, kbId: string): string[];
@@ -97,3 +114,21 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
97
114
  }>;
98
115
  export declare function getChunkCount(db: Database.Database, kbId: string): number;
99
116
  export declare function getFileCount(db: Database.Database, kbId: string): number;
117
+ export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
118
+ export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
119
+ export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
120
+ export declare function searchSymbols(db: Database.Database, query: string, options?: {
121
+ kbId?: string;
122
+ kind?: KnowledgeSymbol["kind"];
123
+ filePattern?: string;
124
+ limit?: number;
125
+ offset?: number;
126
+ exact?: boolean;
127
+ }): KnowledgeSymbol[];
128
+ export declare function countSymbols(db: Database.Database, query: string, options?: {
129
+ kbId?: string;
130
+ kind?: KnowledgeSymbol["kind"];
131
+ filePattern?: string;
132
+ exact?: boolean;
133
+ }): number;
134
+ export declare function getSymbolCount(db: Database.Database, kbId: string): number;
@@ -4,7 +4,7 @@ import { createRequire } from "node:module";
4
4
  import { homedir } from "node:os";
5
5
  import { basename, dirname, join } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
- const SCHEMA_VERSION = 3;
7
+ const SCHEMA_VERSION = 4;
8
8
  const ITERATION_BATCH_SIZE = 500;
9
9
  const SCHEMA_SQL = `
10
10
  CREATE TABLE IF NOT EXISTS schema_version (
@@ -64,6 +64,28 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
64
64
  CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
65
65
  CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
66
66
 
67
+ CREATE TABLE IF NOT EXISTS symbols (
68
+ id TEXT PRIMARY KEY,
69
+ kb_id TEXT NOT NULL,
70
+ name TEXT NOT NULL,
71
+ normalized_name TEXT NOT NULL,
72
+ kind TEXT NOT NULL,
73
+ file_path TEXT NOT NULL,
74
+ file_type TEXT NOT NULL DEFAULT 'text',
75
+ start_line INTEGER NOT NULL DEFAULT 0,
76
+ end_line INTEGER NOT NULL DEFAULT 0,
77
+ container_name TEXT,
78
+ signature TEXT,
79
+ text TEXT NOT NULL DEFAULT '',
80
+ metadata_json TEXT NOT NULL DEFAULT '{}',
81
+ indexed_at INTEGER NOT NULL,
82
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
83
+ );
84
+
85
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
86
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
87
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
88
+
67
89
  CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
68
90
  content_tokenized,
69
91
  content=chunks,
@@ -239,6 +261,31 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
239
261
  }
240
262
  continue;
241
263
  }
264
+ if (v === 4) {
265
+ db.exec(`
266
+ CREATE TABLE IF NOT EXISTS symbols (
267
+ id TEXT PRIMARY KEY,
268
+ kb_id TEXT NOT NULL,
269
+ name TEXT NOT NULL,
270
+ normalized_name TEXT NOT NULL,
271
+ kind TEXT NOT NULL,
272
+ file_path TEXT NOT NULL,
273
+ file_type TEXT NOT NULL DEFAULT 'text',
274
+ start_line INTEGER NOT NULL DEFAULT 0,
275
+ end_line INTEGER NOT NULL DEFAULT 0,
276
+ container_name TEXT,
277
+ signature TEXT,
278
+ text TEXT NOT NULL DEFAULT '',
279
+ metadata_json TEXT NOT NULL DEFAULT '{}',
280
+ indexed_at INTEGER NOT NULL,
281
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
282
+ );
283
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
284
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
285
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
286
+ `);
287
+ continue;
288
+ }
242
289
  if (migrations[v])
243
290
  db.exec(migrations[v]);
244
291
  }
@@ -265,6 +312,7 @@ export function listKBs(db) {
265
312
  }
266
313
  export function deleteKB(db, id) {
267
314
  db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
315
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(id);
268
316
  db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
269
317
  db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
270
318
  }
@@ -332,15 +380,19 @@ export function getIndexingJob(db, kbId) {
332
380
  }
333
381
  // --- CRUD: Chunks ---
334
382
  export function insertChunks(db, kbId, chunks) {
383
+ const ids = [];
335
384
  const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
336
385
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
337
386
  const now = Date.now();
338
387
  const insertMany = db.transaction((items) => {
339
388
  for (const c of items) {
340
- stmt.run(randomUUID(), kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
389
+ const id = randomUUID();
390
+ ids.push(id);
391
+ stmt.run(id, kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
341
392
  }
342
393
  });
343
394
  insertMany(chunks);
395
+ return ids;
344
396
  }
345
397
  export function getChunksByKB(db, kbId) {
346
398
  return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
@@ -447,3 +499,108 @@ export function getFileCount(db, kbId) {
447
499
  const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
448
500
  return row.count;
449
501
  }
502
+ // --- CRUD: Symbols ---
503
+ function normalizeSymbolName(name) {
504
+ return name.trim().toLowerCase();
505
+ }
506
+ function escapeLike(value) {
507
+ return value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
508
+ }
509
+ export function insertSymbols(db, kbId, symbols) {
510
+ if (symbols.length === 0)
511
+ return;
512
+ const stmt = db.prepare(`INSERT INTO symbols (
513
+ id, kb_id, name, normalized_name, kind, file_path, file_type, start_line, end_line,
514
+ container_name, signature, text, metadata_json, indexed_at
515
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
516
+ const now = Date.now();
517
+ const insertMany = db.transaction((items) => {
518
+ for (const symbol of items) {
519
+ stmt.run(randomUUID(), kbId, symbol.name, normalizeSymbolName(symbol.name), symbol.kind, symbol.file_path, symbol.file_type, symbol.start_line, symbol.end_line, symbol.container_name ?? null, symbol.signature ?? null, symbol.text, symbol.metadata_json, now);
520
+ }
521
+ });
522
+ insertMany(symbols);
523
+ }
524
+ export function deleteSymbolsByKB(db, kbId) {
525
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
526
+ }
527
+ export function deleteSymbolsByFile(db, kbId, filePath) {
528
+ db.prepare("DELETE FROM symbols WHERE kb_id = ? AND file_path = ?").run(kbId, filePath);
529
+ }
530
+ export function searchSymbols(db, query, options = {}) {
531
+ const clauses = ["kb.status IN ('ready', 'stale')"];
532
+ const params = [];
533
+ const normalizedQuery = normalizeSymbolName(query);
534
+ if (options.kbId) {
535
+ clauses.push("s.kb_id = ?");
536
+ params.push(options.kbId);
537
+ }
538
+ if (options.kind) {
539
+ clauses.push("s.kind = ?");
540
+ params.push(options.kind);
541
+ }
542
+ if (options.filePattern) {
543
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
544
+ params.push(`%${escapeLike(options.filePattern)}%`);
545
+ }
546
+ if (normalizedQuery) {
547
+ if (options.exact) {
548
+ clauses.push("s.normalized_name = ?");
549
+ params.push(normalizedQuery);
550
+ }
551
+ else {
552
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
553
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
554
+ params.push(pattern, pattern, pattern);
555
+ }
556
+ }
557
+ const where = `WHERE ${clauses.join(" AND ")}`;
558
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
559
+ const offset = Math.max(0, options.offset ?? 0);
560
+ return db
561
+ .prepare(`SELECT s.* FROM symbols s
562
+ JOIN knowledge_bases kb ON kb.id = s.kb_id
563
+ ${where}
564
+ ORDER BY
565
+ CASE WHEN s.normalized_name = ? THEN 0 WHEN s.normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END,
566
+ s.file_path, s.start_line
567
+ LIMIT ? OFFSET ?`)
568
+ .all(...params, normalizedQuery, `${escapeLike(normalizedQuery)}%`, limit, offset);
569
+ }
570
+ export function countSymbols(db, query, options = {}) {
571
+ const clauses = ["kb.status IN ('ready', 'stale')"];
572
+ const params = [];
573
+ const normalizedQuery = normalizeSymbolName(query);
574
+ if (options.kbId) {
575
+ clauses.push("s.kb_id = ?");
576
+ params.push(options.kbId);
577
+ }
578
+ if (options.kind) {
579
+ clauses.push("s.kind = ?");
580
+ params.push(options.kind);
581
+ }
582
+ if (options.filePattern) {
583
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
584
+ params.push(`%${escapeLike(options.filePattern)}%`);
585
+ }
586
+ if (normalizedQuery) {
587
+ if (options.exact) {
588
+ clauses.push("s.normalized_name = ?");
589
+ params.push(normalizedQuery);
590
+ }
591
+ else {
592
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
593
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
594
+ params.push(pattern, pattern, pattern);
595
+ }
596
+ }
597
+ const where = `WHERE ${clauses.join(" AND ")}`;
598
+ const row = db
599
+ .prepare(`SELECT COUNT(*) as count FROM symbols s JOIN knowledge_bases kb ON kb.id = s.kb_id ${where}`)
600
+ .get(...params);
601
+ return row.count;
602
+ }
603
+ export function getSymbolCount(db, kbId) {
604
+ const row = db.prepare("SELECT COUNT(*) as count FROM symbols WHERE kb_id = ?").get(kbId);
605
+ return row.count;
606
+ }
@@ -1,4 +1,5 @@
1
- export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void;
1
+ import { type ScanOptions } from "../indexer/chunker.ts";
2
+ export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
2
3
  export declare function stopWatcher(kbId: string): void;
3
4
  export declare function stopAllWatchers(): void;
4
5
  export declare function getActiveWatcherCount(): number;
@@ -1,5 +1,5 @@
1
- import { existsSync, readdirSync, statSync, watch } from "node:fs";
2
- import { join } from "node:path";
1
+ import { existsSync, statSync, watch } from "node:fs";
2
+ import { createSkippedScanStats, iterateScannableFiles } from "../indexer/chunker.js";
3
3
  const watchers = new Map();
4
4
  const pollers = new Map();
5
5
  const snapshots = new Map();
@@ -15,34 +15,20 @@ function scheduleUpdate(kbId, onUpdate) {
15
15
  onUpdate(kbId);
16
16
  }, DEBOUNCE_MS));
17
17
  }
18
- function scanSnapshot(dirPath) {
18
+ function scanSnapshot(dirPath, options = {}) {
19
19
  const snapshot = new Map();
20
- function scan(dir) {
21
- let entries;
20
+ if (!existsSync(dirPath))
21
+ return snapshot;
22
+ const skipped = createSkippedScanStats();
23
+ for (const file of iterateScannableFiles(dirPath, skipped, options)) {
22
24
  try {
23
- entries = readdirSync(dir, { withFileTypes: true });
25
+ const stat = statSync(file.path);
26
+ snapshot.set(file.path, `${stat.mtimeMs}:${stat.size}`);
24
27
  }
25
28
  catch {
26
- return;
27
- }
28
- for (const entry of entries) {
29
- const fullPath = join(dir, entry.name);
30
- try {
31
- const stat = statSync(fullPath);
32
- if (entry.isDirectory()) {
33
- scan(fullPath);
34
- }
35
- else if (entry.isFile()) {
36
- snapshot.set(fullPath, `${stat.mtimeMs}:${stat.size}`);
37
- }
38
- }
39
- catch {
40
- /* file disappeared or is unreadable */
41
- }
29
+ /* file disappeared or is unreadable */
42
30
  }
43
31
  }
44
- if (existsSync(dirPath))
45
- scan(dirPath);
46
32
  return snapshot;
47
33
  }
48
34
  function snapshotsDiffer(a, b) {
@@ -54,23 +40,27 @@ function snapshotsDiffer(a, b) {
54
40
  }
55
41
  return false;
56
42
  }
57
- function startPoller(kbId, dirPath, onUpdate) {
58
- snapshots.set(kbId, scanSnapshot(dirPath));
43
+ function startPoller(kbId, dirPath, onUpdate, options = {}) {
44
+ snapshots.set(kbId, scanSnapshot(dirPath, options));
59
45
  pollers.set(kbId, setInterval(() => {
60
46
  const previous = snapshots.get(kbId) ?? new Map();
61
- const next = scanSnapshot(dirPath);
47
+ const next = scanSnapshot(dirPath, options);
62
48
  if (snapshotsDiffer(previous, next)) {
63
49
  snapshots.set(kbId, next);
64
50
  scheduleUpdate(kbId, onUpdate);
65
51
  }
66
52
  }, POLL_MS));
67
53
  }
68
- export function startWatcher(kbId, dirPath, onUpdate) {
54
+ export function startWatcher(kbId, dirPath, onUpdate, options = {}) {
69
55
  stopWatcher(kbId);
70
- startPoller(kbId, dirPath, onUpdate);
56
+ startPoller(kbId, dirPath, onUpdate, options);
71
57
  try {
72
58
  const watcher = watch(dirPath, { recursive: true }, () => {
73
- snapshots.set(kbId, scanSnapshot(dirPath));
59
+ const previous = snapshots.get(kbId) ?? new Map();
60
+ const next = scanSnapshot(dirPath, options);
61
+ if (!snapshotsDiffer(previous, next))
62
+ return;
63
+ snapshots.set(kbId, next);
74
64
  scheduleUpdate(kbId, onUpdate);
75
65
  });
76
66
  watcher.on("error", () => {
@@ -26,8 +26,8 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
26
26
  | `OPENAI_BASE_URL` | `https://api.openai.com/v1` | Common OpenAI-compatible API root fallback. |
27
27
  | `PI_KNOWLEDGE_EMBEDDING_MAX_CHARS` | `20000` | Final per-input API embedding safety cap for OpenAI-compatible servers with smaller context windows. This does not replace chunker bounds. |
28
28
  | `PI_KNOWLEDGE_EMBEDDING_API_FALLBACK` | unset | Set to `local` to explicitly fall back to local embeddings after API failures. Without this, API failures are surfaced. |
29
- | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Set to `true` to opt into native ONNX idle disposal. Disabled by default for stable shutdown. |
30
- | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle-dispose timer used only when native idle disposal is enabled. Mainly for lifecycle stress tests. |
29
+ | `PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE` | unset | Enables idle timers for embedding/reranker run coordination. It does not guarantee immediate worker memory release; model-worker shutdown remains tied to engine/session shutdown. Disabled by default for stable shutdown. |
30
+ | `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
31
31
  | `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
32
32
 
33
33
  API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended.
@@ -40,6 +40,12 @@ API embedding failures intentionally surface by default. Silent fallback can hid
40
40
  | `PI_KNOWLEDGE_AUTO_INJECT` | unset | Set to `true` to auto-search KB context before model calls. This is opt-in. |
41
41
  | `PI_KNOWLEDGE_STALE_INDEXING_MS` | built-in stale threshold | Override the stale indexing threshold used by diagnostics and `knowledge_doctor`. |
42
42
 
43
+ Search diagnostics include result provenance in both text output and structured `details`: chunk id, chunk hash, match reason, indexed timestamp, stale flag, and source mtime when the original source file is available. This provenance is stored in SQLite and derived from indexed source metadata; it does not create or modify files in the indexed project.
44
+
45
+ `knowledge_doctor` emits both human-readable issues and machine-readable actions in structured `details`. Action codes include `run_update`, `rebuild_kb`, `wait_for_indexing`, `review_skipped_scope`, `rebuild_vectors`, `check_source`, and `none`. Agents should prefer these action codes over parsing the English action text.
46
+
47
+ `knowledge_symbol_search` uses a lightweight symbol index stored in `knowledge.db`. It is rebuilt during `knowledge_add`, `knowledge_update`, and `knowledge_import`, and can find code symbols, route-like handlers, Markdown headings, config keys, and environment variables. Older file/directory/URL KBs without symbol metadata should be updated before relying on symbol lookup. Imported portable KBs and inline text KBs do not retain an active source path; rebuild their symbols by re-importing or re-adding the content.
48
+
43
49
  ## Test and Release Fixtures
44
50
 
45
51
  | Variable | Default | Purpose |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-knowledge",
3
- "version": "0.4.7",
3
+ "version": "0.5.0",
4
4
  "description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",