pi-knowledge 0.4.6 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ import type { KnowledgeSymbolInsert } from "../storage/sqlite.ts";
2
+ export declare function extractSymbols(content: string, filePath: string, fileType: string): KnowledgeSymbolInsert[];
@@ -0,0 +1,175 @@
1
+ import { extname } from "node:path";
2
+ const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java"]);
3
+ const CONFIG_EXTENSIONS = new Set([".json", ".yaml", ".yml", ".toml", ".env"]);
4
+ function lineNumberAt(content, offset) {
5
+ let line = 1;
6
+ for (let i = 0; i < offset; i++) {
7
+ if (content.charCodeAt(i) === 10)
8
+ line++;
9
+ }
10
+ return line;
11
+ }
12
+ function lineEnd(content, startLine, maxLines = 12) {
13
+ return Math.min(startLine + maxLines, content.split("\n").length);
14
+ }
15
+ function makeSymbol(name, kind, filePath, fileType, startLine, options = {}) {
16
+ const trimmed = name.trim();
17
+ if (!trimmed)
18
+ return undefined;
19
+ return {
20
+ name: trimmed,
21
+ kind,
22
+ file_path: filePath,
23
+ file_type: fileType,
24
+ start_line: startLine,
25
+ end_line: options.endLine ?? startLine,
26
+ container_name: options.containerName ?? null,
27
+ signature: options.signature ?? null,
28
+ text: options.text ?? trimmed,
29
+ metadata_json: JSON.stringify(options.metadata ?? {}),
30
+ };
31
+ }
32
+ function pushSymbol(symbols, symbol) {
33
+ if (symbol)
34
+ symbols.push(symbol);
35
+ }
36
+ function extractMarkdownSymbols(content, filePath, fileType) {
37
+ const symbols = [];
38
+ const lines = content.split("\n");
39
+ for (let index = 0; index < lines.length; index++) {
40
+ const match = /^(#{1,6})\s+(.+?)\s*$/.exec(lines[index]);
41
+ if (!match)
42
+ continue;
43
+ pushSymbol(symbols, makeSymbol(match[2].replace(/\s+#*$/, ""), "heading", filePath, fileType, index + 1, {
44
+ text: lines[index],
45
+ metadata: { depth: match[1].length },
46
+ }));
47
+ }
48
+ return symbols;
49
+ }
50
+ function extractConfigSymbols(content, filePath, fileType) {
51
+ const symbols = [];
52
+ const lines = content.split("\n");
53
+ const ext = extname(filePath).toLowerCase();
54
+ for (let index = 0; index < lines.length; index++) {
55
+ const line = lines[index];
56
+ if (ext === ".json") {
57
+ const match = /^\s*"([^"]+)"\s*:/.exec(line);
58
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
59
+ continue;
60
+ }
61
+ if (ext === ".yaml" || ext === ".yml") {
62
+ const match = /^\s*([A-Za-z0-9_.-]+)\s*:/.exec(line);
63
+ pushSymbol(symbols, match ? makeSymbol(match[1], "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
64
+ continue;
65
+ }
66
+ if (ext === ".toml") {
67
+ const section = /^\s*\[+([A-Za-z0-9_.-]+)\]+/.exec(line);
68
+ const key = /^\s*([A-Za-z0-9_.-]+)\s*=/.exec(line);
69
+ const name = section?.[1] ?? key?.[1];
70
+ pushSymbol(symbols, name ? makeSymbol(name, "config_key", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
71
+ continue;
72
+ }
73
+ const env = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
74
+ pushSymbol(symbols, env ? makeSymbol(env[1], "env_var", filePath, fileType, index + 1, { text: line.trim() }) : undefined);
75
+ }
76
+ return symbols;
77
+ }
78
+ function extractCodeSymbols(content, filePath, fileType) {
79
+ const symbols = [];
80
+ const patterns = [
81
+ {
82
+ regex: /\b(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\([^)]*\)/g,
83
+ kind: "function",
84
+ nameIndex: 1,
85
+ },
86
+ { regex: /\b(?:export\s+)?class\s+([A-Za-z_$][\w$]*)\b/g, kind: "class", nameIndex: 1 },
87
+ { regex: /\b(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/g, kind: "interface", nameIndex: 1 },
88
+ { regex: /\b(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/g, kind: "type", nameIndex: 1 },
89
+ {
90
+ regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/g,
91
+ kind: "function",
92
+ nameIndex: 1,
93
+ },
94
+ { regex: /\b(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/g, kind: "variable", nameIndex: 1 },
95
+ { regex: /\bfunc\s+(?:\([^)]+\)\s*)?([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
96
+ { regex: /\btype\s+([A-Za-z_][\w]*)\s+(?:struct|interface)\b/g, kind: "type", nameIndex: 1 },
97
+ { regex: /\bdef\s+([A-Za-z_][\w]*)\s*\([^)]*\)\s*:/g, kind: "function", nameIndex: 1 },
98
+ { regex: /\bclass\s+([A-Za-z_][\w]*)\b/g, kind: "class", nameIndex: 1 },
99
+ { regex: /\bfn\s+([A-Za-z_][\w]*)\s*\([^)]*\)/g, kind: "function", nameIndex: 1 },
100
+ { regex: /\b(?:struct|enum|trait)\s+([A-Za-z_][\w]*)\b/g, kind: "type", nameIndex: 1 },
101
+ ];
102
+ const seen = new Set();
103
+ for (const pattern of patterns) {
104
+ for (const match of content.matchAll(pattern.regex)) {
105
+ const name = match[pattern.nameIndex];
106
+ const index = match.index ?? 0;
107
+ const startLine = lineNumberAt(content, index);
108
+ const key = `${pattern.kind}:${name}:${startLine}`;
109
+ if (seen.has(key))
110
+ continue;
111
+ seen.add(key);
112
+ pushSymbol(symbols, makeSymbol(name, pattern.kind, filePath, fileType, startLine, {
113
+ endLine: lineEnd(content, startLine),
114
+ signature: match[0].split("\n")[0].trim(),
115
+ text: match[0].split("\n")[0].trim(),
116
+ }));
117
+ }
118
+ }
119
+ return symbols;
120
+ }
121
+ function extractRouteSymbols(content, filePath, fileType) {
122
+ const symbols = [];
123
+ const patterns = [
124
+ {
125
+ regex: /\b(?:app|router|route|server|api|hono)\s*\.\s*(get|post|put|patch|delete|del|options|head|all|use)\s*\(\s*["'`]([^"'`]+)["'`]/g,
126
+ methodIndex: 1,
127
+ pathIndex: 2,
128
+ },
129
+ {
130
+ regex: /\b(?:http\.)?HandleFunc\s*\(\s*["'`]([^"'`]+)["'`]/g,
131
+ pathIndex: 1,
132
+ defaultMethod: "ANY",
133
+ },
134
+ {
135
+ regex: /\b(?:http\.)?Handle\s*\(\s*["'`]([^"'`]+)["'`]/g,
136
+ pathIndex: 1,
137
+ defaultMethod: "ANY",
138
+ },
139
+ ];
140
+ const seen = new Set();
141
+ for (const pattern of patterns) {
142
+ for (const match of content.matchAll(pattern.regex)) {
143
+ const rawMethod = pattern.methodIndex ? match[pattern.methodIndex] : pattern.defaultMethod;
144
+ const method = rawMethod === "del" ? "DELETE" : (rawMethod ?? "ANY").toUpperCase();
145
+ const path = match[pattern.pathIndex];
146
+ const index = match.index ?? 0;
147
+ const startLine = lineNumberAt(content, index);
148
+ const name = `${method} ${path}`;
149
+ const key = `${name}:${startLine}`;
150
+ if (seen.has(key))
151
+ continue;
152
+ seen.add(key);
153
+ pushSymbol(symbols, makeSymbol(name, "route", filePath, fileType, startLine, {
154
+ signature: match[0].split("\n")[0].trim(),
155
+ text: match[0].split("\n")[0].trim(),
156
+ metadata: { method, path },
157
+ }));
158
+ }
159
+ }
160
+ return symbols;
161
+ }
162
+ export function extractSymbols(content, filePath, fileType) {
163
+ const ext = extname(filePath).toLowerCase();
164
+ const symbols = [];
165
+ if (fileType === "markdown" || ext === ".md" || ext === ".mdx")
166
+ symbols.push(...extractMarkdownSymbols(content, filePath, fileType));
167
+ if (CONFIG_EXTENSIONS.has(ext) || filePath.split("/").pop()?.startsWith(".env")) {
168
+ symbols.push(...extractConfigSymbols(content, filePath, fileType));
169
+ }
170
+ if (CODE_EXTENSIONS.has(ext) || ["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
171
+ symbols.push(...extractCodeSymbols(content, filePath, fileType));
172
+ symbols.push(...extractRouteSymbols(content, filePath, fileType));
173
+ }
174
+ return symbols;
175
+ }
@@ -3,7 +3,7 @@ export interface RerankWorkerCandidate {
3
3
  chunkId: string;
4
4
  content: string;
5
5
  }
6
- export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number): Promise<Array<{
6
+ export declare function rerankInModelWorker(query: string, candidates: RerankWorkerCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
7
7
  chunkId: string;
8
8
  score: number;
9
9
  }>>;
@@ -86,19 +86,19 @@ async function requestModelWorker(message, signal) {
86
86
  const id = nextRequestId++;
87
87
  return new Promise((resolve, reject) => {
88
88
  let abortHandler;
89
+ const cleanup = () => {
90
+ if (abortHandler)
91
+ signal?.removeEventListener("abort", abortHandler);
92
+ };
89
93
  pending.set(id, { resolve, reject });
90
94
  if (signal) {
91
95
  abortHandler = () => {
92
96
  pending.delete(id);
97
+ cleanup();
93
98
  reject(new Error("Cancelled"));
94
- shutdownModelWorker();
95
99
  };
96
100
  signal.addEventListener("abort", abortHandler, { once: true });
97
101
  }
98
- const cleanup = () => {
99
- if (abortHandler)
100
- signal?.removeEventListener("abort", abortHandler);
101
- };
102
102
  const originalResolve = resolve;
103
103
  const originalReject = reject;
104
104
  pending.set(id, {
@@ -130,8 +130,8 @@ export async function embedInModelWorker(texts, prefix, signal) {
130
130
  return new Float32Array(vector);
131
131
  });
132
132
  }
133
- export async function rerankInModelWorker(query, candidates, topK) {
134
- const result = await requestModelWorker({ type: "rerank", query, candidates, topK });
133
+ export async function rerankInModelWorker(query, candidates, topK, signal) {
134
+ const result = await requestModelWorker({ type: "rerank", query, candidates, topK }, signal);
135
135
  if (!Array.isArray(result))
136
136
  throw new Error("Invalid reranker worker response");
137
137
  return result.map((item) => {
@@ -4,7 +4,7 @@ export interface RerankCandidate {
4
4
  chunkId: string;
5
5
  content: string;
6
6
  }
7
- export declare function rerank(query: string, candidates: RerankCandidate[], topK: number): Promise<Array<{
7
+ export declare function rerank(query: string, candidates: RerankCandidate[], topK: number, signal?: AbortSignal): Promise<Array<{
8
8
  chunkId: string;
9
9
  score: number;
10
10
  }>>;
@@ -53,12 +53,12 @@ export async function prepareRerankerForShutdown() {
53
53
  clearTimer();
54
54
  await waitForNoActiveRuns();
55
55
  }
56
- export async function rerank(query, candidates, topK) {
56
+ export async function rerank(query, candidates, topK, signal) {
57
57
  if (candidates.length === 0)
58
58
  return [];
59
59
  beginRun();
60
60
  try {
61
- return await rerankInModelWorker(query, candidates, topK);
61
+ return await rerankInModelWorker(query, candidates, topK, signal);
62
62
  }
63
63
  finally {
64
64
  endRun();
@@ -10,5 +10,5 @@ type ChunkIdSource = Iterable<string | {
10
10
  id: string;
11
11
  }>;
12
12
  export declare function searchVector(queryVec: Float32Array, vectors: Float32Array[], chunkIds: string[], limit?: number): VectorResult[];
13
- export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number): VectorFileSearchResult;
13
+ export declare function searchVectorFile(queryVec: Float32Array, vectorPath: string, chunkIds: ChunkIdSource, limit?: number, signal?: AbortSignal): VectorFileSearchResult;
14
14
  export {};
@@ -25,7 +25,9 @@ export function searchVector(queryVec, vectors, chunkIds, limit = 50) {
25
25
  scores.sort((a, b) => b.score - a.score);
26
26
  return scores.slice(0, limit);
27
27
  }
28
- export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
28
+ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50, signal) {
29
+ if (signal?.aborted)
30
+ throw new Error("Cancelled");
29
31
  const reader = openVectorReader(vectorPath);
30
32
  if (!reader)
31
33
  return { results: [], vectorsByChunkId: new Map() };
@@ -36,6 +38,8 @@ export function searchVectorFile(queryVec, vectorPath, chunkIds, limit = 50) {
36
38
  const top = [];
37
39
  let i = 0;
38
40
  for (const chunkIdRef of chunkIds) {
41
+ if (signal?.aborted)
42
+ throw new Error("Cancelled");
39
43
  if (i >= reader.count)
40
44
  break;
41
45
  const vectorIndex = i;
@@ -45,6 +45,23 @@ export interface Chunk {
45
45
  indexed_at: number;
46
46
  }
47
47
  export type ChunkInsert = Omit<Chunk, "id" | "kb_id" | "indexed_at">;
48
+ export interface KnowledgeSymbol {
49
+ id: string;
50
+ kb_id: string;
51
+ name: string;
52
+ normalized_name: string;
53
+ kind: "function" | "class" | "interface" | "type" | "variable" | "heading" | "config_key" | "env_var" | "route";
54
+ file_path: string;
55
+ file_type: string;
56
+ start_line: number;
57
+ end_line: number;
58
+ container_name: string | null;
59
+ signature: string | null;
60
+ text: string;
61
+ metadata_json: string;
62
+ indexed_at: number;
63
+ }
64
+ export type KnowledgeSymbolInsert = Omit<KnowledgeSymbol, "id" | "kb_id" | "indexed_at" | "normalized_name">;
48
65
  export declare function getDefaultKnowledgeDir(): string;
49
66
  export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
50
67
  legacyPiDir?: string;
@@ -78,7 +95,7 @@ export declare function updateIndexingJob(db: Database.Database, kbId: string, p
78
95
  }): void;
79
96
  export declare function finishIndexingJob(db: Database.Database, kbId: string, status: Extract<IndexingJob["status"], "succeeded" | "failed" | "cancelled">, message: string, errorMessage?: string): void;
80
97
  export declare function getIndexingJob(db: Database.Database, kbId: string): IndexingJob | undefined;
81
- export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): void;
98
+ export declare function insertChunks(db: Database.Database, kbId: string, chunks: ChunkInsert[]): string[];
82
99
  export declare function getChunksByKB(db: Database.Database, kbId: string): Chunk[];
83
100
  export declare function iterateChunksByKB(db: Database.Database, kbId: string): IterableIterator<Chunk>;
84
101
  export declare function getChunkIdsByKB(db: Database.Database, kbId: string): string[];
@@ -97,3 +114,21 @@ export declare function iterateChunkHashesByKB(db: Database.Database, kbId: stri
97
114
  }>;
98
115
  export declare function getChunkCount(db: Database.Database, kbId: string): number;
99
116
  export declare function getFileCount(db: Database.Database, kbId: string): number;
117
+ export declare function insertSymbols(db: Database.Database, kbId: string, symbols: KnowledgeSymbolInsert[]): void;
118
+ export declare function deleteSymbolsByKB(db: Database.Database, kbId: string): void;
119
+ export declare function deleteSymbolsByFile(db: Database.Database, kbId: string, filePath: string): void;
120
+ export declare function searchSymbols(db: Database.Database, query: string, options?: {
121
+ kbId?: string;
122
+ kind?: KnowledgeSymbol["kind"];
123
+ filePattern?: string;
124
+ limit?: number;
125
+ offset?: number;
126
+ exact?: boolean;
127
+ }): KnowledgeSymbol[];
128
+ export declare function countSymbols(db: Database.Database, query: string, options?: {
129
+ kbId?: string;
130
+ kind?: KnowledgeSymbol["kind"];
131
+ filePattern?: string;
132
+ exact?: boolean;
133
+ }): number;
134
+ export declare function getSymbolCount(db: Database.Database, kbId: string): number;
@@ -4,7 +4,8 @@ import { createRequire } from "node:module";
4
4
  import { homedir } from "node:os";
5
5
  import { basename, dirname, join } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
- const SCHEMA_VERSION = 3;
7
+ const SCHEMA_VERSION = 4;
8
+ const ITERATION_BATCH_SIZE = 500;
8
9
  const SCHEMA_SQL = `
9
10
  CREATE TABLE IF NOT EXISTS schema_version (
10
11
  version INTEGER NOT NULL
@@ -63,6 +64,28 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
63
64
  CREATE INDEX IF NOT EXISTS idx_chunks_kb_id ON chunks(kb_id);
64
65
  CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash);
65
66
 
67
+ CREATE TABLE IF NOT EXISTS symbols (
68
+ id TEXT PRIMARY KEY,
69
+ kb_id TEXT NOT NULL,
70
+ name TEXT NOT NULL,
71
+ normalized_name TEXT NOT NULL,
72
+ kind TEXT NOT NULL,
73
+ file_path TEXT NOT NULL,
74
+ file_type TEXT NOT NULL DEFAULT 'text',
75
+ start_line INTEGER NOT NULL DEFAULT 0,
76
+ end_line INTEGER NOT NULL DEFAULT 0,
77
+ container_name TEXT,
78
+ signature TEXT,
79
+ text TEXT NOT NULL DEFAULT '',
80
+ metadata_json TEXT NOT NULL DEFAULT '{}',
81
+ indexed_at INTEGER NOT NULL,
82
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
83
+ );
84
+
85
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
86
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
87
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
88
+
66
89
  CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
67
90
  content_tokenized,
68
91
  content=chunks,
@@ -189,7 +212,7 @@ export function openDatabase(knowledgeDir) {
189
212
  const nativeBinding = isBunRuntime() ? undefined : findBetterSqlite3NativeBinding();
190
213
  const db = nativeBinding ? new Database(dbPath, { nativeBinding }) : new Database(dbPath);
191
214
  applyPragma(db, "journal_mode = WAL");
192
- applyPragma(db, "busy_timeout = 5000");
215
+ applyPragma(db, "busy_timeout = 60000");
193
216
  applyPragma(db, "foreign_keys = ON");
194
217
  const hasVersion = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'").get();
195
218
  if (!hasVersion) {
@@ -238,6 +261,31 @@ CREATE TABLE IF NOT EXISTS indexing_jobs (
238
261
  }
239
262
  continue;
240
263
  }
264
+ if (v === 4) {
265
+ db.exec(`
266
+ CREATE TABLE IF NOT EXISTS symbols (
267
+ id TEXT PRIMARY KEY,
268
+ kb_id TEXT NOT NULL,
269
+ name TEXT NOT NULL,
270
+ normalized_name TEXT NOT NULL,
271
+ kind TEXT NOT NULL,
272
+ file_path TEXT NOT NULL,
273
+ file_type TEXT NOT NULL DEFAULT 'text',
274
+ start_line INTEGER NOT NULL DEFAULT 0,
275
+ end_line INTEGER NOT NULL DEFAULT 0,
276
+ container_name TEXT,
277
+ signature TEXT,
278
+ text TEXT NOT NULL DEFAULT '',
279
+ metadata_json TEXT NOT NULL DEFAULT '{}',
280
+ indexed_at INTEGER NOT NULL,
281
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
282
+ );
283
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_name ON symbols(kb_id, normalized_name);
284
+ CREATE INDEX IF NOT EXISTS idx_symbols_kb_kind ON symbols(kb_id, kind);
285
+ CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(kb_id, file_path);
286
+ `);
287
+ continue;
288
+ }
241
289
  if (migrations[v])
242
290
  db.exec(migrations[v]);
243
291
  }
@@ -264,6 +312,7 @@ export function listKBs(db) {
264
312
  }
265
313
  export function deleteKB(db, id) {
266
314
  db.prepare("DELETE FROM indexing_jobs WHERE kb_id = ?").run(id);
315
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(id);
267
316
  db.prepare("DELETE FROM chunks WHERE kb_id = ?").run(id);
268
317
  db.prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
269
318
  }
@@ -331,28 +380,32 @@ export function getIndexingJob(db, kbId) {
331
380
  }
332
381
  // --- CRUD: Chunks ---
333
382
  export function insertChunks(db, kbId, chunks) {
383
+ const ids = [];
334
384
  const stmt = db.prepare(`INSERT INTO chunks (id, kb_id, content_hash, content, content_tokenized, file_path, file_type, start_line, end_line, metadata_json, indexed_at)
335
385
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
336
386
  const now = Date.now();
337
387
  const insertMany = db.transaction((items) => {
338
388
  for (const c of items) {
339
- stmt.run(randomUUID(), kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
389
+ const id = randomUUID();
390
+ ids.push(id);
391
+ stmt.run(id, kbId, c.content_hash, c.content, c.content_tokenized, c.file_path, c.file_type, c.start_line, c.end_line, c.metadata_json, now);
340
392
  }
341
393
  });
342
394
  insertMany(chunks);
395
+ return ids;
343
396
  }
344
397
  export function getChunksByKB(db, kbId) {
345
398
  return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
346
399
  }
347
400
  export function iterateChunksByKB(db, kbId) {
348
- return db.prepare("SELECT * FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId);
401
+ return iterateChunkRowsByKB(db, kbId);
349
402
  }
350
403
  export function getChunkIdsByKB(db, kbId) {
351
404
  const rows = db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").all(kbId);
352
405
  return rows.map((r) => r.id);
353
406
  }
354
407
  export function iterateChunkIdsByKB(db, kbId) {
355
- return db.prepare("SELECT id FROM chunks WHERE kb_id = ? ORDER BY rowid").iterate(kbId);
408
+ return iterateChunkIdRowsByKB(db, kbId);
356
409
  }
357
410
  export function getChunkById(db, id) {
358
411
  return db.prepare("SELECT * FROM chunks WHERE id = ?").get(id);
@@ -387,9 +440,56 @@ export function getChunkHashesByKB(db, kbId) {
387
440
  return new Map(rows.map((r) => [r.content_hash, r.id]));
388
441
  }
389
442
  export function iterateChunkHashesByKB(db, kbId) {
390
- return db
391
- .prepare("SELECT id, content_hash FROM chunks WHERE kb_id = ? ORDER BY rowid")
392
- .iterate(kbId);
443
+ return iterateChunkHashRowsByKB(db, kbId);
444
+ }
445
+ function* iterateChunkRowsByKB(db, kbId) {
446
+ let lastRowid = 0;
447
+ const statement = db.prepare(`SELECT rowid as rowid, * FROM chunks
448
+ WHERE kb_id = ? AND rowid > ?
449
+ ORDER BY rowid
450
+ LIMIT ?`);
451
+ while (true) {
452
+ const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
453
+ if (rows.length === 0)
454
+ return;
455
+ for (const row of rows) {
456
+ lastRowid = row.rowid;
457
+ const { rowid: _rowid, ...chunk } = row;
458
+ yield chunk;
459
+ }
460
+ }
461
+ }
462
+ function* iterateChunkIdRowsByKB(db, kbId) {
463
+ let lastRowid = 0;
464
+ const statement = db.prepare(`SELECT rowid, id FROM chunks
465
+ WHERE kb_id = ? AND rowid > ?
466
+ ORDER BY rowid
467
+ LIMIT ?`);
468
+ while (true) {
469
+ const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
470
+ if (rows.length === 0)
471
+ return;
472
+ for (const row of rows) {
473
+ lastRowid = row.rowid;
474
+ yield { id: row.id };
475
+ }
476
+ }
477
+ }
478
+ function* iterateChunkHashRowsByKB(db, kbId) {
479
+ let lastRowid = 0;
480
+ const statement = db.prepare(`SELECT rowid, id, content_hash FROM chunks
481
+ WHERE kb_id = ? AND rowid > ?
482
+ ORDER BY rowid
483
+ LIMIT ?`);
484
+ while (true) {
485
+ const rows = statement.all(kbId, lastRowid, ITERATION_BATCH_SIZE);
486
+ if (rows.length === 0)
487
+ return;
488
+ for (const row of rows) {
489
+ lastRowid = row.rowid;
490
+ yield { id: row.id, content_hash: row.content_hash };
491
+ }
492
+ }
393
493
  }
394
494
  export function getChunkCount(db, kbId) {
395
495
  const row = db.prepare("SELECT COUNT(*) as count FROM chunks WHERE kb_id = ?").get(kbId);
@@ -399,3 +499,108 @@ export function getFileCount(db, kbId) {
399
499
  const row = db.prepare("SELECT COUNT(DISTINCT file_path) as count FROM chunks WHERE kb_id = ?").get(kbId);
400
500
  return row.count;
401
501
  }
502
+ // --- CRUD: Symbols ---
503
+ function normalizeSymbolName(name) {
504
+ return name.trim().toLowerCase();
505
+ }
506
+ function escapeLike(value) {
507
+ return value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
508
+ }
509
+ export function insertSymbols(db, kbId, symbols) {
510
+ if (symbols.length === 0)
511
+ return;
512
+ const stmt = db.prepare(`INSERT INTO symbols (
513
+ id, kb_id, name, normalized_name, kind, file_path, file_type, start_line, end_line,
514
+ container_name, signature, text, metadata_json, indexed_at
515
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
516
+ const now = Date.now();
517
+ const insertMany = db.transaction((items) => {
518
+ for (const symbol of items) {
519
+ stmt.run(randomUUID(), kbId, symbol.name, normalizeSymbolName(symbol.name), symbol.kind, symbol.file_path, symbol.file_type, symbol.start_line, symbol.end_line, symbol.container_name ?? null, symbol.signature ?? null, symbol.text, symbol.metadata_json, now);
520
+ }
521
+ });
522
+ insertMany(symbols);
523
+ }
524
+ export function deleteSymbolsByKB(db, kbId) {
525
+ db.prepare("DELETE FROM symbols WHERE kb_id = ?").run(kbId);
526
+ }
527
+ export function deleteSymbolsByFile(db, kbId, filePath) {
528
+ db.prepare("DELETE FROM symbols WHERE kb_id = ? AND file_path = ?").run(kbId, filePath);
529
+ }
530
+ export function searchSymbols(db, query, options = {}) {
531
+ const clauses = ["kb.status IN ('ready', 'stale')"];
532
+ const params = [];
533
+ const normalizedQuery = normalizeSymbolName(query);
534
+ if (options.kbId) {
535
+ clauses.push("s.kb_id = ?");
536
+ params.push(options.kbId);
537
+ }
538
+ if (options.kind) {
539
+ clauses.push("s.kind = ?");
540
+ params.push(options.kind);
541
+ }
542
+ if (options.filePattern) {
543
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
544
+ params.push(`%${escapeLike(options.filePattern)}%`);
545
+ }
546
+ if (normalizedQuery) {
547
+ if (options.exact) {
548
+ clauses.push("s.normalized_name = ?");
549
+ params.push(normalizedQuery);
550
+ }
551
+ else {
552
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
553
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
554
+ params.push(pattern, pattern, pattern);
555
+ }
556
+ }
557
+ const where = `WHERE ${clauses.join(" AND ")}`;
558
+ const limit = Math.max(1, Math.min(200, options.limit ?? 20));
559
+ const offset = Math.max(0, options.offset ?? 0);
560
+ return db
561
+ .prepare(`SELECT s.* FROM symbols s
562
+ JOIN knowledge_bases kb ON kb.id = s.kb_id
563
+ ${where}
564
+ ORDER BY
565
+ CASE WHEN s.normalized_name = ? THEN 0 WHEN s.normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END,
566
+ s.file_path, s.start_line
567
+ LIMIT ? OFFSET ?`)
568
+ .all(...params, normalizedQuery, `${escapeLike(normalizedQuery)}%`, limit, offset);
569
+ }
570
+ export function countSymbols(db, query, options = {}) {
571
+ const clauses = ["kb.status IN ('ready', 'stale')"];
572
+ const params = [];
573
+ const normalizedQuery = normalizeSymbolName(query);
574
+ if (options.kbId) {
575
+ clauses.push("s.kb_id = ?");
576
+ params.push(options.kbId);
577
+ }
578
+ if (options.kind) {
579
+ clauses.push("s.kind = ?");
580
+ params.push(options.kind);
581
+ }
582
+ if (options.filePattern) {
583
+ clauses.push("s.file_path LIKE ? ESCAPE '\\'");
584
+ params.push(`%${escapeLike(options.filePattern)}%`);
585
+ }
586
+ if (normalizedQuery) {
587
+ if (options.exact) {
588
+ clauses.push("s.normalized_name = ?");
589
+ params.push(normalizedQuery);
590
+ }
591
+ else {
592
+ clauses.push("(s.normalized_name LIKE ? ESCAPE '\\' OR s.file_path LIKE ? ESCAPE '\\' OR s.text LIKE ? ESCAPE '\\')");
593
+ const pattern = `%${escapeLike(normalizedQuery)}%`;
594
+ params.push(pattern, pattern, pattern);
595
+ }
596
+ }
597
+ const where = `WHERE ${clauses.join(" AND ")}`;
598
+ const row = db
599
+ .prepare(`SELECT COUNT(*) as count FROM symbols s JOIN knowledge_bases kb ON kb.id = s.kb_id ${where}`)
600
+ .get(...params);
601
+ return row.count;
602
+ }
603
+ export function getSymbolCount(db, kbId) {
604
+ const row = db.prepare("SELECT COUNT(*) as count FROM symbols WHERE kb_id = ?").get(kbId);
605
+ return row.count;
606
+ }
@@ -1,4 +1,5 @@
1
- export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void): void;
1
+ import { type ScanOptions } from "../indexer/chunker.ts";
2
+ export declare function startWatcher(kbId: string, dirPath: string, onUpdate: (kbId: string) => void, options?: ScanOptions): void;
2
3
  export declare function stopWatcher(kbId: string): void;
3
4
  export declare function stopAllWatchers(): void;
4
5
  export declare function getActiveWatcherCount(): number;