pi-knowledge 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engine.ts ADDED
@@ -0,0 +1,300 @@
1
+ import { existsSync, statSync } from "node:fs";
2
+ import { join, resolve } from "node:path";
3
+ import type Database from "better-sqlite3";
4
+ import { embedDocuments, embedQuery, dispose as disposeEmbedding } from "./embedding/provider.ts";
5
+ import { loadVectors, saveVectors } from "./embedding/vectors.ts";
6
+ import { chunkFile, walkDir } from "./indexer/chunker.ts";
7
+ import { searchBM25 } from "./search/bm25.ts";
8
+ import { reciprocalRankFusion } from "./search/fusion.ts";
9
+ import { rerank, disposeReranker } from "./search/reranker.ts";
10
+ import { searchVector } from "./search/vector.ts";
11
+ import { diagnoseKB, type DiagnosticResult } from "./diagnostics/health.ts";
12
+ import {
13
+ createKB, deleteKB, getChunkById, getChunkIdsByKB, getChunksByKB, getFileCount, getKB, getKBByName,
14
+ insertChunks, listKBs, openDatabase, updateKBCounts, updateKBStatus, deleteChunksByIds, getChunkHashesByKB,
15
+ type KnowledgeBase,
16
+ } from "./storage/sqlite.ts";
17
+
18
+ export interface SearchOptions {
19
+ mode?: "fast" | "semantic" | "hybrid" | "deep";
20
+ limit?: number;
21
+ offset?: number;
22
+ kb_id?: string;
23
+ filters?: { file_type?: string; path_pattern?: string };
24
+ }
25
+
26
+ export interface SearchResult {
27
+ content: string;
28
+ file_path: string;
29
+ file_type: string;
30
+ kb_name: string;
31
+ score: number;
32
+ snippet: string;
33
+ start_line: number;
34
+ end_line: number;
35
+ }
36
+
37
+ export interface SearchResponse {
38
+ results: SearchResult[];
39
+ total_count: number;
40
+ has_more: boolean;
41
+ }
42
+
43
+ export type ProgressCallback = (msg: string) => void;
44
+
45
+ export class KnowledgeEngine {
46
+ private db: Database.Database | null = null;
47
+ private knowledgeDir: string = "";
48
+
49
+ async initialize(knowledgeDir: string): Promise<void> {
50
+ this.knowledgeDir = knowledgeDir;
51
+ this.db = openDatabase(knowledgeDir);
52
+ }
53
+
54
+ async add(source: string, name: string, onProgress?: ProgressCallback): Promise<{ kb: KnowledgeBase; chunkCount: number }> {
55
+ if (!this.db) throw new Error("Engine not initialized");
56
+ const resolvedSource = resolve(source);
57
+ const isDir = existsSync(resolvedSource) && statSync(resolvedSource).isDirectory();
58
+ const isFile = existsSync(resolvedSource) && statSync(resolvedSource).isFile();
59
+ const sourceType = isDir ? "directory" : isFile ? "file" : "text";
60
+
61
+ const kb = createKB(this.db, { name, source_path: isDir || isFile ? resolvedSource : undefined, source_type: sourceType });
62
+ updateKBStatus(this.db, kb.id, "indexing");
63
+
64
+ try {
65
+ let allChunks: Awaited<ReturnType<typeof chunkFile>> = [];
66
+
67
+ if (isDir) {
68
+ onProgress?.(`Scanning ${resolvedSource}...`);
69
+ const files = walkDir(resolvedSource);
70
+ onProgress?.(`Found ${files.length} files, chunking...`);
71
+ for (const file of files) {
72
+ allChunks.push(...await chunkFile(file.content, file.relPath));
73
+ }
74
+ } else if (isFile) {
75
+ const { readFileSync } = await import("node:fs");
76
+ const content = readFileSync(resolvedSource, "utf-8");
77
+ allChunks = await chunkFile(content, resolvedSource);
78
+ } else {
79
+ allChunks = await chunkFile(source, "inline-text");
80
+ }
81
+
82
+ onProgress?.(`${allChunks.length} chunks, embedding...`);
83
+ const texts = allChunks.map((c) => c.content);
84
+ const vectors = await embedDocuments(texts);
85
+
86
+ onProgress?.(`Storing...`);
87
+ insertChunks(this.db, kb.id, allChunks);
88
+
89
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
90
+ saveVectors(vectorPath, vectors);
91
+
92
+ const fileCount = isDir ? getFileCount(this.db, kb.id) : 1;
93
+ updateKBCounts(this.db, kb.id, allChunks.length, fileCount);
94
+ updateKBStatus(this.db, kb.id, "ready");
95
+
96
+ return { kb: getKB(this.db, kb.id)!, chunkCount: allChunks.length };
97
+ } catch (e) {
98
+ // Clean up partial state on failure
99
+ deleteKB(this.db, kb.id);
100
+ throw e;
101
+ }
102
+ }
103
+
104
+ async update(nameOrId: string, onProgress?: ProgressCallback): Promise<{ added: number; removed: number; unchanged: number }> {
105
+ if (!this.db) throw new Error("Engine not initialized");
106
+ const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
107
+ if (!kb) throw new Error(`Knowledge base not found: ${nameOrId}`);
108
+ if (!kb.source_path || !existsSync(kb.source_path)) {
109
+ throw new Error(`Source path not available or missing: ${kb.source_path}`);
110
+ }
111
+
112
+ updateKBStatus(this.db, kb.id, "indexing");
113
+
114
+ try {
115
+ // 1. Scan and chunk current source
116
+ onProgress?.("Scanning source...");
117
+ const isDir = statSync(kb.source_path).isDirectory();
118
+ let newChunks: Awaited<ReturnType<typeof chunkFile>> = [];
119
+ if (isDir) {
120
+ const files = walkDir(kb.source_path);
121
+ for (const file of files) newChunks.push(...await chunkFile(file.content, file.relPath));
122
+ } else {
123
+ const { readFileSync } = await import("node:fs");
124
+ newChunks = await chunkFile(readFileSync(kb.source_path, "utf-8"), kb.source_path);
125
+ }
126
+
127
+ // 2. Load existing state: chunks (hash→id) + vectors (ordered)
128
+ const existingHashes = getChunkHashesByKB(this.db, kb.id);
129
+ const existingChunkIds = getChunkIdsByKB(this.db, kb.id);
130
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
131
+ const existingVectors = loadVectors(vectorPath);
132
+
133
+ // Build hash→vector cache from existing data
134
+ const vectorCache = new Map<string, Float32Array>();
135
+ const existingChunks = getChunksByKB(this.db, kb.id);
136
+ for (let i = 0; i < existingChunks.length; i++) {
137
+ if (existingVectors[i]) vectorCache.set(existingChunks[i].content_hash, existingVectors[i]);
138
+ }
139
+
140
+ // 3. Identify changes
141
+ const newHashSet = new Set(newChunks.map((c) => c.content_hash));
142
+ const chunksToAdd = newChunks.filter((c) => !existingHashes.has(c.content_hash));
143
+ const idsToRemove = [...existingHashes.entries()].filter(([hash]) => !newHashSet.has(hash)).map(([, id]) => id);
144
+ const unchanged = newChunks.length - chunksToAdd.length;
145
+
146
+ onProgress?.(`Changes: +${chunksToAdd.length} -${idsToRemove.length} =${unchanged}`);
147
+
148
+ // 4. Remove deleted chunks from DB
149
+ if (idsToRemove.length > 0) deleteChunksByIds(this.db, idsToRemove);
150
+
151
+ // 5. Embed ONLY new chunks
152
+ let newVectors: Float32Array[] = [];
153
+ if (chunksToAdd.length > 0) {
154
+ onProgress?.(`Embedding ${chunksToAdd.length} new chunks...`);
155
+ newVectors = await embedDocuments(chunksToAdd.map((c) => c.content));
156
+ insertChunks(this.db, kb.id, chunksToAdd);
157
+ }
158
+
159
+ // Add new vectors to cache
160
+ for (let i = 0; i < chunksToAdd.length; i++) {
161
+ vectorCache.set(chunksToAdd[i].content_hash, newVectors[i]);
162
+ }
163
+
164
+ // 6. Rebuild vector file in DB chunk order (reusing cached vectors)
165
+ const finalChunks = getChunksByKB(this.db, kb.id);
166
+ const finalVectors = finalChunks.map((c) => vectorCache.get(c.content_hash)!).filter(Boolean);
167
+ saveVectors(vectorPath, finalVectors);
168
+
169
+ // 7. Update counts
170
+ updateKBCounts(this.db, kb.id, finalChunks.length, getFileCount(this.db, kb.id));
171
+ updateKBStatus(this.db, kb.id, "ready");
172
+
173
+ return { added: chunksToAdd.length, removed: idsToRemove.length, unchanged };
174
+ } catch (e) {
175
+ updateKBStatus(this.db, kb.id, "error");
176
+ throw e;
177
+ }
178
+ }
179
+
180
+ async search(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
181
+ if (!this.db) throw new Error("Engine not initialized");
182
+ const { mode = "hybrid", limit = 10, offset = 0, kb_id, filters } = options;
183
+
184
+ const kbs = kb_id ? [getKB(this.db, kb_id)].filter(Boolean) as KnowledgeBase[] : listKBs(this.db);
185
+ if (kbs.length === 0) return { results: [], total_count: 0, has_more: false };
186
+
187
+ let allResults: { chunkId: string; score: number }[] = [];
188
+
189
+ for (const kb of kbs) {
190
+ const chunkIds = getChunkIdsByKB(this.db, kb.id);
191
+ if (chunkIds.length === 0) continue;
192
+
193
+ if (mode === "fast") {
194
+ allResults.push(...searchBM25(this.db, query, 50, kb.id));
195
+ } else if (mode === "semantic") {
196
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
197
+ const vectors = loadVectors(vectorPath);
198
+ if (vectors.length > 0) {
199
+ const queryVec = await embedQuery(query);
200
+ allResults.push(...searchVector(queryVec, vectors, chunkIds));
201
+ }
202
+ } else {
203
+ // hybrid: BM25 + vector + RRF (both scoped to this KB)
204
+ const bm25Results = searchBM25(this.db, query, 50, kb.id);
205
+ const vectorPath = join(this.knowledgeDir, "vectors", `${kb.id}.bin`);
206
+ const vectors = loadVectors(vectorPath);
207
+ let vecResults: { chunkId: string; score: number }[] = [];
208
+ if (vectors.length > 0) {
209
+ const queryVec = await embedQuery(query);
210
+ vecResults = searchVector(queryVec, vectors, chunkIds);
211
+ }
212
+ const fused = reciprocalRankFusion([bm25Results, vecResults]);
213
+ allResults.push(...fused);
214
+ }
215
+ }
216
+
217
+ // Deduplicate and sort
218
+ const seen = new Set<string>();
219
+ const unique = allResults.filter((r) => { if (seen.has(r.chunkId)) return false; seen.add(r.chunkId); return true; });
220
+ unique.sort((a, b) => b.score - a.score);
221
+
222
+ // Apply metadata filters post-retrieval
223
+ let filtered = unique;
224
+ if (filters?.file_type || filters?.path_pattern) {
225
+ filtered = unique.filter((r) => {
226
+ const chunk = getChunkById(this.db!, r.chunkId);
227
+ if (!chunk) return false;
228
+ if (filters.file_type && chunk.file_type !== filters.file_type) return false;
229
+ if (filters.path_pattern && !chunk.file_path.includes(filters.path_pattern)) return false;
230
+ return true;
231
+ });
232
+ }
233
+
234
+ // Deep mode: rerank top-20 with cross-encoder
235
+ if (mode === "deep" && filtered.length > 0) {
236
+ const candidates = filtered.slice(0, 20).map((r) => {
237
+ const chunk = getChunkById(this.db!, r.chunkId);
238
+ return chunk ? { chunkId: r.chunkId, content: chunk.content } : null;
239
+ }).filter(Boolean) as Array<{ chunkId: string; content: string }>;
240
+ const reranked = await rerank(query, candidates, limit);
241
+ const results: SearchResult[] = reranked.map((r) => {
242
+ const chunk = getChunkById(this.db!, r.chunkId)!;
243
+ const kbObj = getKB(this.db!, chunk.kb_id);
244
+ return { content: chunk.content, file_path: chunk.file_path, file_type: chunk.file_type, kb_name: kbObj?.name ?? "unknown", score: r.score, snippet: chunk.content.slice(0, 200), start_line: chunk.start_line, end_line: chunk.end_line };
245
+ });
246
+ return { results, total_count: results.length, has_more: false };
247
+ }
248
+
249
+ const total = filtered.length;
250
+ const page = filtered.slice(offset, offset + limit);
251
+
252
+ const results: SearchResult[] = page.map((r) => {
253
+ const chunk = getChunkById(this.db!, r.chunkId);
254
+ if (!chunk) return null;
255
+ const kb = getKB(this.db!, chunk.kb_id);
256
+ return {
257
+ content: chunk.content,
258
+ file_path: chunk.file_path,
259
+ file_type: chunk.file_type,
260
+ kb_name: kb?.name ?? "unknown",
261
+ score: r.score,
262
+ snippet: chunk.content.slice(0, 200),
263
+ start_line: chunk.start_line,
264
+ end_line: chunk.end_line,
265
+ };
266
+ }).filter(Boolean) as SearchResult[];
267
+
268
+ return { results, total_count: total, has_more: offset + limit < total };
269
+ }
270
+
271
+ remove(nameOrId: string): boolean {
272
+ if (!this.db) return false;
273
+ const kb = getKB(this.db, nameOrId) ?? getKBByName(this.db, nameOrId);
274
+ if (!kb) return false;
275
+ deleteKB(this.db, kb.id);
276
+ return true;
277
+ }
278
+
279
+ list(): KnowledgeBase[] {
280
+ if (!this.db) return [];
281
+ return listKBs(this.db);
282
+ }
283
+
284
+ clear(): void {
285
+ if (!this.db) return;
286
+ for (const kb of listKBs(this.db)) deleteKB(this.db, kb.id);
287
+ }
288
+
289
+ diagnose(): DiagnosticResult[] {
290
+ if (!this.db) return [];
291
+ return listKBs(this.db).map((kb) => diagnoseKB(this.db!, kb));
292
+ }
293
+
294
+ async dispose(): Promise<void> {
295
+ await disposeEmbedding();
296
+ await disposeReranker();
297
+ this.db?.close();
298
+ this.db = null;
299
+ }
300
+ }
@@ -0,0 +1,263 @@
1
+ import { createHash } from "node:crypto";
2
+ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
3
+ import { extname, join, relative, sep } from "node:path";
4
+ import ignore from "ignore";
5
+ import type { ChunkInsert } from "../storage/sqlite.ts";
6
+
7
+ const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
8
+
9
+ const BINARY_EXTENSIONS = new Set([
10
+ ".png", ".jpg", ".jpeg", ".gif", ".ico", ".bmp", ".webp", ".svg",
11
+ ".woff", ".woff2", ".ttf", ".eot", ".otf",
12
+ ".zip", ".gz", ".tar", ".bz2", ".7z", ".rar",
13
+ ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt",
14
+ ".exe", ".dll", ".so", ".dylib", ".node",
15
+ ".db", ".sqlite", ".bin", ".dat", ".lock",
16
+ ".mp3", ".mp4", ".wav", ".avi", ".mov", ".webm",
17
+ ".wasm", ".o", ".a", ".lib",
18
+ ]);
19
+
20
+ const DEFAULT_IGNORE = [
21
+ "node_modules", ".git", "dist", "build", ".next", "__pycache__",
22
+ "*.min.js", "*.min.css", "*.map", "*.lock", "package-lock.json",
23
+ ".DS_Store", "Thumbs.db", "*.pyc", "*.class",
24
+ ];
25
+
26
+ export interface ScannedFile {
27
+ path: string; // absolute path
28
+ relPath: string; // relative to root
29
+ content: string;
30
+ fileType: string;
31
+ }
32
+
33
+ function detectFileType(filePath: string): string {
34
+ const ext = extname(filePath).toLowerCase();
35
+ const map: Record<string, string> = {
36
+ ".ts": "typescript", ".tsx": "typescript",
37
+ ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript",
38
+ ".py": "python", ".go": "go", ".rs": "rust", ".java": "java",
39
+ ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
40
+ ".md": "markdown", ".mdx": "markdown",
41
+ ".json": "json", ".yaml": "yaml", ".yml": "yaml", ".toml": "toml",
42
+ ".html": "html", ".css": "css", ".scss": "css",
43
+ ".sh": "shell", ".bash": "shell", ".zsh": "shell",
44
+ ".sql": "sql", ".graphql": "graphql",
45
+ ".txt": "text", ".csv": "text", ".log": "text",
46
+ };
47
+ return map[ext] ?? "text";
48
+ }
49
+
50
+ function isBinaryFile(filePath: string): boolean {
51
+ if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
52
+ try {
53
+ const fd = readFileSync(filePath, { flag: "r" });
54
+ const sample = fd.subarray(0, 512);
55
+ return sample.includes(0x00);
56
+ } catch {
57
+ return true;
58
+ }
59
+ }
60
+
61
+ export function walkDir(dirPath: string): ScannedFile[] {
62
+ const ig = ignore();
63
+ ig.add(DEFAULT_IGNORE);
64
+
65
+ const gitignorePath = join(dirPath, ".gitignore");
66
+ if (existsSync(gitignorePath)) {
67
+ ig.add(readFileSync(gitignorePath, "utf-8"));
68
+ }
69
+
70
+ const results: ScannedFile[] = [];
71
+
72
+ function walk(dir: string): void {
73
+ const entries = readdirSync(dir, { withFileTypes: true });
74
+ for (const entry of entries) {
75
+ const fullPath = join(dir, entry.name);
76
+ const relPath = relative(dirPath, fullPath).split(sep).join("/");
77
+
78
+ if (ig.ignores(relPath)) continue;
79
+
80
+ if (entry.isDirectory()) {
81
+ if (ig.ignores(relPath + "/")) continue;
82
+ walk(fullPath);
83
+ } else if (entry.isFile()) {
84
+ const stat = statSync(fullPath);
85
+ if (stat.size > MAX_FILE_SIZE) continue;
86
+ if (isBinaryFile(fullPath)) continue;
87
+
88
+ try {
89
+ const content = readFileSync(fullPath, "utf-8");
90
+ results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
91
+ } catch {
92
+ // skip unreadable files
93
+ }
94
+ }
95
+ }
96
+ }
97
+
98
+ walk(dirPath);
99
+ return results;
100
+ }
101
+
102
+ // --- Chunking ---
103
+
104
+ function estimateTokens(text: string): number {
105
+ return Math.ceil(text.length / 3);
106
+ }
107
+
108
+ export function preTokenizeForFTS(content: string): string {
109
+ return content
110
+ .replace(/([a-z])([A-Z])/g, "$1 $2")
111
+ .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
112
+ .replace(/([a-zA-Z])(\d)/g, "$1 $2")
113
+ .replace(/(\d)([a-zA-Z])/g, "$1 $2")
114
+ .replace(/([\u4e00-\u9fff\u3400-\u4dbf])/g, " $1 ")
115
+ .replace(/\s+/g, " ")
116
+ .trim();
117
+ }
118
+
119
+ export function contentHash(content: string): string {
120
+ return createHash("sha256").update(content).digest("hex");
121
+ }
122
+
123
+ export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
124
+ const lines = content.split("\n");
125
+ const chunks: Omit<ChunkInsert, "kb_id">[] = [];
126
+ let currentHeading = "";
127
+ let sectionLines: string[] = [];
128
+ let startLine = 1;
129
+
130
+ function flush(endLine: number): void {
131
+ const text = sectionLines.join("\n").trim();
132
+ if (text.length < 50) return;
133
+
134
+ const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
135
+
136
+ if (estimateTokens(fullText) > 2000) {
137
+ // Split large sections by paragraph
138
+ const paragraphs = fullText.split(/\n\n+/);
139
+ let buf: string[] = [];
140
+ let bufStart = startLine;
141
+ for (const para of paragraphs) {
142
+ buf.push(para);
143
+ if (estimateTokens(buf.join("\n\n")) > 1000) {
144
+ const chunkText = buf.join("\n\n");
145
+ chunks.push({
146
+ content_hash: contentHash(chunkText),
147
+ content: chunkText,
148
+ content_tokenized: preTokenizeForFTS(chunkText),
149
+ file_path: filePath,
150
+ file_type: "markdown",
151
+ start_line: bufStart,
152
+ end_line: endLine,
153
+ metadata_json: JSON.stringify({ heading: currentHeading }),
154
+ });
155
+ buf = [];
156
+ bufStart = endLine;
157
+ }
158
+ }
159
+ if (buf.length > 0) {
160
+ const chunkText = buf.join("\n\n");
161
+ if (chunkText.length >= 50) {
162
+ chunks.push({
163
+ content_hash: contentHash(chunkText),
164
+ content: chunkText,
165
+ content_tokenized: preTokenizeForFTS(chunkText),
166
+ file_path: filePath,
167
+ file_type: "markdown",
168
+ start_line: bufStart,
169
+ end_line: endLine,
170
+ metadata_json: JSON.stringify({ heading: currentHeading }),
171
+ });
172
+ }
173
+ }
174
+ } else {
175
+ chunks.push({
176
+ content_hash: contentHash(fullText),
177
+ content: fullText,
178
+ content_tokenized: preTokenizeForFTS(fullText),
179
+ file_path: filePath,
180
+ file_type: "markdown",
181
+ start_line: startLine,
182
+ end_line: endLine,
183
+ metadata_json: JSON.stringify({ heading: currentHeading }),
184
+ });
185
+ }
186
+ }
187
+
188
+ for (let i = 0; i < lines.length; i++) {
189
+ const line = lines[i];
190
+ const headingMatch = line.match(/^(#{1,4})\s+(.+)$/);
191
+
192
+ if (headingMatch && sectionLines.length > 0) {
193
+ flush(i);
194
+ sectionLines = [];
195
+ startLine = i + 1;
196
+ }
197
+
198
+ if (headingMatch) {
199
+ currentHeading = line;
200
+ } else {
201
+ sectionLines.push(line);
202
+ }
203
+ }
204
+
205
+ if (sectionLines.length > 0) {
206
+ flush(lines.length);
207
+ }
208
+
209
+ return chunks;
210
+ }
211
+
212
+ export function chunkText(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
213
+ const fileType = detectFileType(filePath);
214
+ const paragraphs = content.split(/\n\n+/);
215
+ const chunks: Omit<ChunkInsert, "kb_id">[] = [];
216
+ let buffer: string[] = [];
217
+ let bufferTokens = 0;
218
+ let lineOffset = 1;
219
+
220
+ function flush(): void {
221
+ const text = buffer.join("\n\n").trim();
222
+ if (text.length < 50) return;
223
+ chunks.push({
224
+ content_hash: contentHash(text),
225
+ content: text,
226
+ content_tokenized: preTokenizeForFTS(text),
227
+ file_path: filePath,
228
+ file_type: fileType,
229
+ start_line: lineOffset,
230
+ end_line: lineOffset + text.split("\n").length,
231
+ metadata_json: "{}",
232
+ });
233
+ lineOffset += text.split("\n").length + 1;
234
+ }
235
+
236
+ for (const para of paragraphs) {
237
+ const paraTokens = estimateTokens(para);
238
+ if (bufferTokens + paraTokens > 1000 && buffer.length > 0) {
239
+ flush();
240
+ // overlap: keep last paragraph
241
+ buffer = buffer.length > 0 ? [buffer[buffer.length - 1]] : [];
242
+ bufferTokens = buffer.length > 0 ? estimateTokens(buffer[0]) : 0;
243
+ }
244
+ buffer.push(para);
245
+ bufferTokens += paraTokens;
246
+ }
247
+
248
+ if (buffer.length > 0) flush();
249
+ return chunks;
250
+ }
251
+
252
+ export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
253
+ const fileType = detectFileType(filePath);
254
+ if (fileType === "markdown") return chunkMarkdown(content, filePath);
255
+ if (["typescript", "javascript", "python", "go", "rust"].includes(fileType)) {
256
+ try {
257
+ const { chunkWithAST } = await import("./chunkers/code-ast.ts");
258
+ const astChunks = await chunkWithAST(content, filePath, fileType);
259
+ if (astChunks.length > 0) return astChunks;
260
+ } catch { /* fallback to text chunker */ }
261
+ }
262
+ return chunkText(content, filePath);
263
+ }
@@ -0,0 +1,139 @@
1
+ import { contentHash, preTokenizeForFTS } from "../chunker.ts";
2
+ import type { ChunkInsert } from "../../storage/sqlite.ts";
3
+
4
+ interface ASTNode {
5
+ type: string;
6
+ text: string;
7
+ startPosition: { row: number };
8
+ endPosition: { row: number };
9
+ children: ASTNode[];
10
+ childForFieldName(name: string): ASTNode | null;
11
+ }
12
+
13
+ type LangConfig = {
14
+ grammar: () => Promise<any>;
15
+ fnTypes: Set<string>;
16
+ classTypes: Set<string>;
17
+ methodContainer?: string;
18
+ classBody?: string;
19
+ fileType: string;
20
+ };
21
+
22
+ const LANGS: Record<string, LangConfig> = {
23
+ typescript: {
24
+ grammar: async () => { const m = await import("tree-sitter-typescript"); return m.typescript ?? m.default?.typescript; },
25
+ fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
26
+ classTypes: new Set(["class_declaration", "interface_declaration"]),
27
+ classBody: "class_body",
28
+ fileType: "typescript",
29
+ },
30
+ javascript: {
31
+ grammar: async () => { const m = await import("tree-sitter-typescript"); return m.typescript ?? m.default?.typescript; },
32
+ fnTypes: new Set(["function_declaration", "arrow_function", "generator_function_declaration"]),
33
+ classTypes: new Set(["class_declaration"]),
34
+ classBody: "class_body",
35
+ fileType: "javascript",
36
+ },
37
+ python: {
38
+ grammar: async () => { const m = await import("tree-sitter-python"); return m.default ?? m; },
39
+ fnTypes: new Set(["function_definition"]),
40
+ classTypes: new Set(["class_definition"]),
41
+ classBody: "block",
42
+ fileType: "python",
43
+ },
44
+ go: {
45
+ grammar: async () => { const m = await import("tree-sitter-go"); return m.default ?? m; },
46
+ fnTypes: new Set(["function_declaration", "method_declaration"]),
47
+ classTypes: new Set(["type_declaration"]),
48
+ fileType: "go",
49
+ },
50
+ rust: {
51
+ grammar: async () => { const m = await import("tree-sitter-rust"); return m.default ?? m; },
52
+ fnTypes: new Set(["function_item"]),
53
+ classTypes: new Set(["struct_item", "enum_item"]),
54
+ methodContainer: "impl_item",
55
+ fileType: "rust",
56
+ },
57
+ };
58
+
59
+ function getName(node: ASTNode): string {
60
+ return node.childForFieldName("name")?.text ?? node.childForFieldName("type")?.text ?? "anonymous";
61
+ }
62
+
63
+ function collectChunks(root: ASTNode, config: LangConfig): Array<{ name: string; text: string; start: number; end: number }> {
64
+ const chunks: Array<{ name: string; text: string; start: number; end: number }> = [];
65
+
66
+ function walk(node: ASTNode) {
67
+ if (node.type === "export_statement") {
68
+ for (const child of node.children) {
69
+ if (config.fnTypes.has(child.type)) { chunks.push({ name: getName(child), text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 }); return; }
70
+ if (config.classTypes.has(child.type)) { walk(child); return; }
71
+ }
72
+ }
73
+ if (node.type === "decorated_definition") {
74
+ for (const child of node.children) {
75
+ if (config.fnTypes.has(child.type) || config.classTypes.has(child.type)) {
76
+ chunks.push({ name: getName(child), text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 }); return;
77
+ }
78
+ }
79
+ }
80
+ if (config.fnTypes.has(node.type)) {
81
+ chunks.push({ name: getName(node), text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 }); return;
82
+ }
83
+ if (config.classTypes.has(node.type)) {
84
+ const name = getName(node);
85
+ let hasChildren = false;
86
+ for (const child of node.children) {
87
+ if (config.classBody && child.type === config.classBody) {
88
+ for (const m of child.children) {
89
+ if (config.fnTypes.has(m.type) || m.type === "method_definition") {
90
+ hasChildren = true;
91
+ chunks.push({ name: `${name}.${getName(m)}`, text: m.text, start: m.startPosition.row + 1, end: m.endPosition.row + 1 });
92
+ }
93
+ }
94
+ }
95
+ }
96
+ if (!hasChildren) chunks.push({ name, text: node.text, start: node.startPosition.row + 1, end: node.endPosition.row + 1 });
97
+ return;
98
+ }
99
+ if (config.methodContainer && node.type === config.methodContainer) {
100
+ const typeName = getName(node);
101
+ for (const child of node.children) {
102
+ if (child.type === "declaration_list") {
103
+ for (const m of child.children) {
104
+ if (config.fnTypes.has(m.type)) chunks.push({ name: `${typeName}.${getName(m)}`, text: m.text, start: m.startPosition.row + 1, end: m.endPosition.row + 1 });
105
+ }
106
+ }
107
+ }
108
+ return;
109
+ }
110
+ for (const child of node.children) walk(child);
111
+ }
112
+
113
+ walk(root);
114
+ return chunks;
115
+ }
116
+
117
+ export async function chunkWithAST(content: string, filePath: string, language: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
118
+ const config = LANGS[language];
119
+ if (!config) return [];
120
+ const Parser = (await import("tree-sitter")).default;
121
+ const grammar = await config.grammar();
122
+ const parser = new Parser();
123
+ parser.setLanguage(grammar);
124
+ const tree = parser.parse(content);
125
+ const fns = collectChunks(tree.rootNode as unknown as ASTNode, config);
126
+ if (fns.length === 0) return [];
127
+ return fns.map((fn) => ({
128
+ content_hash: contentHash(fn.text),
129
+ content: fn.text,
130
+ content_tokenized: preTokenizeForFTS(fn.text),
131
+ file_path: filePath,
132
+ file_type: config.fileType,
133
+ start_line: fn.start,
134
+ end_line: fn.end,
135
+ metadata_json: JSON.stringify({ function_name: fn.name }),
136
+ }));
137
+ }
138
+
139
+ export const SUPPORTED_LANGUAGES = new Set(Object.keys(LANGS));