pi-knowledge 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +29 -8
- package/dist/index.d.ts +38 -0
- package/dist/index.js +435 -0
- package/dist/src/diagnostics/health.d.ts +19 -0
- package/dist/src/diagnostics/health.js +110 -0
- package/dist/src/embedding/provider.d.ts +5 -0
- package/dist/src/embedding/provider.js +99 -0
- package/dist/src/embedding/vectors.d.ts +15 -0
- package/dist/src/embedding/vectors.js +111 -0
- package/dist/src/engine.d.ts +90 -0
- package/dist/src/engine.js +1370 -0
- package/dist/src/indexer/chunker.d.ts +53 -0
- package/dist/src/indexer/chunker.js +498 -0
- package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
- package/dist/src/indexer/chunkers/code-ast.js +179 -0
- package/dist/src/model-worker-client.d.ts +10 -0
- package/dist/src/model-worker-client.js +138 -0
- package/dist/src/model-worker.d.ts +1 -0
- package/dist/src/model-worker.js +61 -0
- package/dist/src/search/bm25.d.ts +8 -0
- package/dist/src/search/bm25.js +38 -0
- package/dist/src/search/fusion.d.ts +19 -0
- package/dist/src/search/fusion.js +34 -0
- package/dist/src/search/query.d.ts +5 -0
- package/dist/src/search/query.js +57 -0
- package/dist/src/search/ranking.d.ts +27 -0
- package/dist/src/search/ranking.js +180 -0
- package/dist/src/search/reranker.d.ts +10 -0
- package/dist/src/search/reranker.js +66 -0
- package/dist/src/search/vector.d.ts +14 -0
- package/dist/src/search/vector.js +58 -0
- package/dist/src/storage/sqlite.d.ts +99 -0
- package/dist/src/storage/sqlite.js +330 -0
- package/dist/src/watcher/file-watcher.d.ts +4 -0
- package/dist/src/watcher/file-watcher.js +108 -0
- package/extension.js +9 -0
- package/package.json +20 -6
- package/index.ts +0 -500
- package/src/diagnostics/health.ts +0 -145
- package/src/embedding/provider.ts +0 -106
- package/src/embedding/vectors.ts +0 -119
- package/src/engine.ts +0 -1592
- package/src/indexer/chunker.ts +0 -612
- package/src/indexer/chunkers/code-ast.ts +0 -209
- package/src/model-worker-client.ts +0 -146
- package/src/model-worker.ts +0 -88
- package/src/search/bm25.ts +0 -55
- package/src/search/fusion.ts +0 -48
- package/src/search/query.ts +0 -59
- package/src/search/ranking.ts +0 -201
- package/src/search/reranker.ts +0 -74
- package/src/search/vector.ts +0 -81
- package/src/storage/sqlite.ts +0 -489
- package/src/watcher/file-watcher.ts +0 -116
package/src/indexer/chunker.ts
DELETED
|
@@ -1,612 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
|
|
3
|
-
import { extname, join, relative, sep } from "node:path";
|
|
4
|
-
import ignore from "ignore";
|
|
5
|
-
import type { ChunkInsert } from "../storage/sqlite.ts";
|
|
6
|
-
|
|
7
|
-
const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
|
|
8
|
-
|
|
9
|
-
const BINARY_EXTENSIONS = new Set([
|
|
10
|
-
".png",
|
|
11
|
-
".jpg",
|
|
12
|
-
".jpeg",
|
|
13
|
-
".gif",
|
|
14
|
-
".ico",
|
|
15
|
-
".bmp",
|
|
16
|
-
".webp",
|
|
17
|
-
".svg",
|
|
18
|
-
".woff",
|
|
19
|
-
".woff2",
|
|
20
|
-
".ttf",
|
|
21
|
-
".eot",
|
|
22
|
-
".otf",
|
|
23
|
-
".zip",
|
|
24
|
-
".gz",
|
|
25
|
-
".tar",
|
|
26
|
-
".bz2",
|
|
27
|
-
".7z",
|
|
28
|
-
".rar",
|
|
29
|
-
".pdf",
|
|
30
|
-
".doc",
|
|
31
|
-
".docx",
|
|
32
|
-
".xls",
|
|
33
|
-
".xlsx",
|
|
34
|
-
".ppt",
|
|
35
|
-
".exe",
|
|
36
|
-
".dll",
|
|
37
|
-
".so",
|
|
38
|
-
".dylib",
|
|
39
|
-
".node",
|
|
40
|
-
".db",
|
|
41
|
-
".sqlite",
|
|
42
|
-
".bin",
|
|
43
|
-
".dat",
|
|
44
|
-
".lock",
|
|
45
|
-
".mp3",
|
|
46
|
-
".mp4",
|
|
47
|
-
".wav",
|
|
48
|
-
".avi",
|
|
49
|
-
".mov",
|
|
50
|
-
".webm",
|
|
51
|
-
".wasm",
|
|
52
|
-
".o",
|
|
53
|
-
".a",
|
|
54
|
-
".lib",
|
|
55
|
-
]);
|
|
56
|
-
|
|
57
|
-
const DEFAULT_SUGGESTED_EXCLUDE = [
|
|
58
|
-
"node_modules",
|
|
59
|
-
".git",
|
|
60
|
-
"dist",
|
|
61
|
-
"build",
|
|
62
|
-
"bin",
|
|
63
|
-
"obj",
|
|
64
|
-
"out",
|
|
65
|
-
"target",
|
|
66
|
-
"coverage",
|
|
67
|
-
".next",
|
|
68
|
-
".cache",
|
|
69
|
-
".playwright",
|
|
70
|
-
".browser",
|
|
71
|
-
".browsers",
|
|
72
|
-
"__pycache__",
|
|
73
|
-
"*.app",
|
|
74
|
-
"*.asar",
|
|
75
|
-
"*.asar.unpacked",
|
|
76
|
-
"*.pak",
|
|
77
|
-
".env",
|
|
78
|
-
".env.*",
|
|
79
|
-
"*.pem",
|
|
80
|
-
"*.key",
|
|
81
|
-
"*.p12",
|
|
82
|
-
"*.pfx",
|
|
83
|
-
"*.crt",
|
|
84
|
-
"*.cert",
|
|
85
|
-
"*secret*",
|
|
86
|
-
"*secrets*",
|
|
87
|
-
"*credential*",
|
|
88
|
-
"*credentials*",
|
|
89
|
-
"docs/*eval-report*.md",
|
|
90
|
-
"docs/*evaluation-report*.md",
|
|
91
|
-
"docs/*knowledge-base*report*.md",
|
|
92
|
-
"*knowledge-base-evaluation-report*.md",
|
|
93
|
-
"*knowledge*.jsonl",
|
|
94
|
-
"*.min.js",
|
|
95
|
-
"*.min.css",
|
|
96
|
-
"*.map",
|
|
97
|
-
"*.lock",
|
|
98
|
-
"package-lock.json",
|
|
99
|
-
"playwright-report",
|
|
100
|
-
"test-results",
|
|
101
|
-
"browser-cache",
|
|
102
|
-
"ms-playwright",
|
|
103
|
-
".DS_Store",
|
|
104
|
-
"Thumbs.db",
|
|
105
|
-
"*.pyc",
|
|
106
|
-
"*.class",
|
|
107
|
-
];
|
|
108
|
-
|
|
109
|
-
export interface ScanOptions {
|
|
110
|
-
includeSuggestedText?: boolean;
|
|
111
|
-
includePaths?: string[];
|
|
112
|
-
excludePaths?: string[];
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export interface ScannedFile {
|
|
116
|
-
path: string; // absolute path
|
|
117
|
-
relPath: string; // relative to root
|
|
118
|
-
content: string;
|
|
119
|
-
fileType: string;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
export interface ScannableFile {
|
|
123
|
-
path: string; // absolute path
|
|
124
|
-
relPath: string; // relative to root
|
|
125
|
-
fileType: string;
|
|
126
|
-
size: number;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export interface SkippedScanEntry {
|
|
130
|
-
path: string;
|
|
131
|
-
reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
|
|
132
|
-
size?: number;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export interface ScanResult {
|
|
136
|
-
files: ScannedFile[];
|
|
137
|
-
skipped: {
|
|
138
|
-
total: number;
|
|
139
|
-
by_reason: Record<SkippedScanEntry["reason"], number>;
|
|
140
|
-
samples: SkippedScanEntry[];
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
const MAX_SKIPPED_SAMPLES = 25;
|
|
145
|
-
|
|
146
|
-
export function createSkippedScanStats(): ScanResult["skipped"] {
|
|
147
|
-
return {
|
|
148
|
-
total: 0,
|
|
149
|
-
by_reason: {
|
|
150
|
-
suggested_excluded: 0,
|
|
151
|
-
oversized: 0,
|
|
152
|
-
binary: 0,
|
|
153
|
-
unreadable: 0,
|
|
154
|
-
inaccessible: 0,
|
|
155
|
-
},
|
|
156
|
-
samples: [],
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
|
|
161
|
-
skipped.total++;
|
|
162
|
-
skipped.by_reason[entry.reason]++;
|
|
163
|
-
if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
|
|
167
|
-
return (
|
|
168
|
-
Object.entries(skipped.by_reason)
|
|
169
|
-
.filter(([, count]) => count > 0)
|
|
170
|
-
.map(([reason, count]) => `${reason}: ${count}`)
|
|
171
|
-
.join(", ") || "none"
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function detectFileType(filePath: string): string {
|
|
176
|
-
const ext = extname(filePath).toLowerCase();
|
|
177
|
-
const map: Record<string, string> = {
|
|
178
|
-
".ts": "typescript",
|
|
179
|
-
".tsx": "typescript",
|
|
180
|
-
".js": "javascript",
|
|
181
|
-
".jsx": "javascript",
|
|
182
|
-
".mjs": "javascript",
|
|
183
|
-
".py": "python",
|
|
184
|
-
".go": "go",
|
|
185
|
-
".rs": "rust",
|
|
186
|
-
".java": "java",
|
|
187
|
-
".c": "c",
|
|
188
|
-
".cpp": "cpp",
|
|
189
|
-
".h": "c",
|
|
190
|
-
".hpp": "cpp",
|
|
191
|
-
".md": "markdown",
|
|
192
|
-
".mdx": "markdown",
|
|
193
|
-
".json": "json",
|
|
194
|
-
".yaml": "yaml",
|
|
195
|
-
".yml": "yaml",
|
|
196
|
-
".toml": "toml",
|
|
197
|
-
".html": "html",
|
|
198
|
-
".css": "css",
|
|
199
|
-
".scss": "css",
|
|
200
|
-
".sh": "shell",
|
|
201
|
-
".bash": "shell",
|
|
202
|
-
".zsh": "shell",
|
|
203
|
-
".sql": "sql",
|
|
204
|
-
".graphql": "graphql",
|
|
205
|
-
".txt": "text",
|
|
206
|
-
".csv": "text",
|
|
207
|
-
".log": "text",
|
|
208
|
-
};
|
|
209
|
-
return map[ext] ?? "text";
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
function isBinaryFile(filePath: string): boolean {
|
|
213
|
-
if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
|
|
214
|
-
let fd: number | undefined;
|
|
215
|
-
try {
|
|
216
|
-
fd = openSync(filePath, "r");
|
|
217
|
-
const sample = Buffer.alloc(512);
|
|
218
|
-
const bytesRead = readSync(fd, sample, 0, sample.length, 0);
|
|
219
|
-
if (bytesRead === 0) return false;
|
|
220
|
-
return sample.subarray(0, bytesRead).includes(0x00);
|
|
221
|
-
} catch {
|
|
222
|
-
return true;
|
|
223
|
-
} finally {
|
|
224
|
-
if (fd !== undefined) closeSync(fd);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
|
|
229
|
-
const ig = ignore();
|
|
230
|
-
ig.add(DEFAULT_SUGGESTED_EXCLUDE);
|
|
231
|
-
|
|
232
|
-
const gitignorePath = join(dirPath, ".gitignore");
|
|
233
|
-
if (existsSync(gitignorePath)) {
|
|
234
|
-
ig.add(readFileSync(gitignorePath, "utf-8"));
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
return ig;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
function normalizeScanPath(path: string): string {
|
|
241
|
-
return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
function normalizeScanPaths(paths: string[] | undefined): string[] {
|
|
245
|
-
return (paths ?? []).map(normalizeScanPath).filter(Boolean);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
function pathMatches(relPath: string, patterns: string[]): boolean {
|
|
249
|
-
const normalized = normalizeScanPath(relPath);
|
|
250
|
-
return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
|
|
254
|
-
const normalized = normalizeScanPath(relPath);
|
|
255
|
-
return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
export function* iterateScannableFiles(
|
|
259
|
-
dirPath: string,
|
|
260
|
-
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
261
|
-
options: ScanOptions = {},
|
|
262
|
-
): Generator<ScannableFile> {
|
|
263
|
-
const ig = buildIgnoreMatcher(dirPath);
|
|
264
|
-
const includePaths = normalizeScanPaths(options.includePaths);
|
|
265
|
-
const excludePaths = normalizeScanPaths(options.excludePaths);
|
|
266
|
-
const includeSuggestedText = options.includeSuggestedText === true;
|
|
267
|
-
|
|
268
|
-
function* walk(dir: string): Generator<ScannableFile> {
|
|
269
|
-
let entries: Dirent[];
|
|
270
|
-
try {
|
|
271
|
-
entries = readdirSync(dir, { withFileTypes: true });
|
|
272
|
-
} catch {
|
|
273
|
-
addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
|
|
274
|
-
return;
|
|
275
|
-
}
|
|
276
|
-
for (const entry of entries) {
|
|
277
|
-
const fullPath = join(dir, entry.name);
|
|
278
|
-
const relPath = relative(dirPath, fullPath).split(sep).join("/");
|
|
279
|
-
const explicitlyIncluded = pathMatches(relPath, includePaths);
|
|
280
|
-
|
|
281
|
-
if (pathMatches(relPath, excludePaths)) {
|
|
282
|
-
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
|
|
287
|
-
if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
288
|
-
yield* walk(fullPath);
|
|
289
|
-
continue;
|
|
290
|
-
}
|
|
291
|
-
addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
|
|
292
|
-
continue;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
if (entry.isDirectory()) {
|
|
296
|
-
if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
|
|
297
|
-
if (directoryCouldContainIncludedPath(relPath, includePaths)) {
|
|
298
|
-
yield* walk(fullPath);
|
|
299
|
-
continue;
|
|
300
|
-
}
|
|
301
|
-
addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
|
|
302
|
-
continue;
|
|
303
|
-
}
|
|
304
|
-
yield* walk(fullPath);
|
|
305
|
-
} else if (entry.isFile()) {
|
|
306
|
-
let size = 0;
|
|
307
|
-
try {
|
|
308
|
-
size = statSync(fullPath).size;
|
|
309
|
-
} catch {
|
|
310
|
-
addSkipped(skipped, { path: relPath, reason: "unreadable" });
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
if (size > MAX_FILE_SIZE) {
|
|
314
|
-
addSkipped(skipped, { path: relPath, reason: "oversized", size });
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
if (isBinaryFile(fullPath)) {
|
|
318
|
-
addSkipped(skipped, { path: relPath, reason: "binary", size });
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
yield* walk(dirPath);
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
export function* iterateScannedFiles(
|
|
331
|
-
dirPath: string,
|
|
332
|
-
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
333
|
-
options: ScanOptions = {},
|
|
334
|
-
): Generator<ScannedFile> {
|
|
335
|
-
for (const file of iterateScannableFiles(dirPath, skipped, options)) {
|
|
336
|
-
try {
|
|
337
|
-
const content = readFileSync(file.path, "utf-8");
|
|
338
|
-
yield { ...file, content };
|
|
339
|
-
} catch {
|
|
340
|
-
addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
export async function iterateScannedFilesAsync(
|
|
346
|
-
dirPath: string,
|
|
347
|
-
onFile: (file: ScannedFile) => Promise<void> | void,
|
|
348
|
-
skipped: ScanResult["skipped"] = createSkippedScanStats(),
|
|
349
|
-
options: ScanOptions = {},
|
|
350
|
-
): Promise<ScanResult["skipped"]> {
|
|
351
|
-
for (const file of iterateScannedFiles(dirPath, skipped, options)) {
|
|
352
|
-
await onFile(file);
|
|
353
|
-
}
|
|
354
|
-
return skipped;
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
|
|
358
|
-
return walkDirDetailed(dirPath, options).files;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
|
|
362
|
-
const results: ScannedFile[] = [];
|
|
363
|
-
const skipped = createSkippedScanStats();
|
|
364
|
-
for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
|
|
365
|
-
return { files: results, skipped };
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
export function isReadableTextFile(filePath: string): boolean {
|
|
369
|
-
return !isBinaryFile(filePath);
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
// --- Chunking ---
|
|
373
|
-
|
|
374
|
-
function estimateTokens(text: string): number {
|
|
375
|
-
return Math.ceil(text.length / 3);
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
const MARKDOWN_TARGET_TOKENS = 450;
|
|
379
|
-
const TEXT_TARGET_TOKENS = 550;
|
|
380
|
-
const MAX_TEXT_CHUNK_CHARS = 6_000;
|
|
381
|
-
|
|
382
|
-
export function preTokenizeForFTS(content: string): string {
|
|
383
|
-
return content
|
|
384
|
-
.replace(/([a-z])([A-Z])/g, "$1 $2")
|
|
385
|
-
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
386
|
-
.replace(/([a-zA-Z])(\d)/g, "$1 $2")
|
|
387
|
-
.replace(/(\d)([a-zA-Z])/g, "$1 $2")
|
|
388
|
-
.replace(/([\u4e00-\u9fff\u3400-\u4dbf])/g, " $1 ")
|
|
389
|
-
.replace(/\s+/g, " ")
|
|
390
|
-
.trim();
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
export function contentHash(content: string): string {
|
|
394
|
-
return createHash("sha256").update(content).digest("hex");
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
export function chunkIdentityHash(opts: {
|
|
398
|
-
content: string;
|
|
399
|
-
filePath: string;
|
|
400
|
-
fileType: string;
|
|
401
|
-
startLine: number;
|
|
402
|
-
endLine: number;
|
|
403
|
-
metadataJson: string;
|
|
404
|
-
}): string {
|
|
405
|
-
return contentHash(
|
|
406
|
-
[opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
|
|
407
|
-
"\0",
|
|
408
|
-
),
|
|
409
|
-
);
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
function normalizeHeading(heading: string): string {
|
|
413
|
-
return heading.replace(/^#{1,6}\s+/, "").trim();
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
function buildContextPrefix(
|
|
417
|
-
filePath: string,
|
|
418
|
-
fileType: string,
|
|
419
|
-
metadata: Record<string, string | number | null | undefined>,
|
|
420
|
-
): string {
|
|
421
|
-
const parts = [`File: ${filePath}`, `Type: ${fileType}`];
|
|
422
|
-
const heading = typeof metadata.heading === "string" ? metadata.heading : "";
|
|
423
|
-
const breadcrumb = typeof metadata.breadcrumb === "string" ? metadata.breadcrumb : "";
|
|
424
|
-
const symbol = typeof metadata.function_name === "string" ? metadata.function_name : "";
|
|
425
|
-
if (breadcrumb) parts.push(`Section: ${breadcrumb}`);
|
|
426
|
-
else if (heading) parts.push(`Section: ${normalizeHeading(heading)}`);
|
|
427
|
-
if (symbol) parts.push(`Symbol: ${symbol}`);
|
|
428
|
-
return parts.join("\n");
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
export function buildChunkEmbeddingText(
|
|
432
|
-
chunk: Pick<ChunkInsert, "content" | "file_path" | "file_type" | "metadata_json">,
|
|
433
|
-
): string {
|
|
434
|
-
let metadata: Record<string, string | number | null | undefined> = {};
|
|
435
|
-
try {
|
|
436
|
-
metadata = JSON.parse(chunk.metadata_json) as Record<string, string | number | null | undefined>;
|
|
437
|
-
} catch {
|
|
438
|
-
metadata = {};
|
|
439
|
-
}
|
|
440
|
-
return `${buildContextPrefix(chunk.file_path, chunk.file_type, metadata)}\n\n${chunk.content}`;
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
function makeChunk(
|
|
444
|
-
content: string,
|
|
445
|
-
filePath: string,
|
|
446
|
-
fileType: string,
|
|
447
|
-
startLine: number,
|
|
448
|
-
endLine: number,
|
|
449
|
-
metadata: Record<string, string | number | null | undefined> = {},
|
|
450
|
-
): Omit<ChunkInsert, "kb_id"> {
|
|
451
|
-
const metadata_json = JSON.stringify(metadata);
|
|
452
|
-
const chunk = {
|
|
453
|
-
content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
|
|
454
|
-
content,
|
|
455
|
-
content_tokenized: "",
|
|
456
|
-
file_path: filePath,
|
|
457
|
-
file_type: fileType,
|
|
458
|
-
start_line: startLine,
|
|
459
|
-
end_line: endLine,
|
|
460
|
-
metadata_json,
|
|
461
|
-
};
|
|
462
|
-
return { ...chunk, content_tokenized: preTokenizeForFTS(buildChunkEmbeddingText(chunk)) };
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
export function chunkMarkdown(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
|
|
466
|
-
const lines = content.split("\n");
|
|
467
|
-
const chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
468
|
-
const headingStack: string[] = [];
|
|
469
|
-
let sectionLines: string[] = [];
|
|
470
|
-
let startLine = 1;
|
|
471
|
-
let currentHeading = "";
|
|
472
|
-
|
|
473
|
-
function currentBreadcrumb(): string {
|
|
474
|
-
return headingStack.map(normalizeHeading).filter(Boolean).join(" > ");
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
function pushMarkdownChunk(text: string, start: number, end: number): void {
|
|
478
|
-
if (text.trim().length < 50) return;
|
|
479
|
-
chunks.push(
|
|
480
|
-
makeChunk(text.trim(), filePath, "markdown", start, end, {
|
|
481
|
-
heading: currentHeading,
|
|
482
|
-
breadcrumb: currentBreadcrumb(),
|
|
483
|
-
}),
|
|
484
|
-
);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
function flush(endLine: number): void {
|
|
488
|
-
const text = sectionLines.join("\n").trim();
|
|
489
|
-
if (text.length < 50) return;
|
|
490
|
-
|
|
491
|
-
const fullText = currentHeading ? `${currentHeading}\n\n${text}` : text;
|
|
492
|
-
const paragraphs = fullText.split(/\n\n+/);
|
|
493
|
-
let buffer: string[] = [];
|
|
494
|
-
let bufferStart = startLine;
|
|
495
|
-
|
|
496
|
-
for (const para of paragraphs) {
|
|
497
|
-
const next = [...buffer, para].join("\n\n");
|
|
498
|
-
if (estimateTokens(next) > MARKDOWN_TARGET_TOKENS && buffer.length > 0) {
|
|
499
|
-
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
500
|
-
buffer = [];
|
|
501
|
-
bufferStart = endLine;
|
|
502
|
-
}
|
|
503
|
-
buffer.push(para);
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
if (buffer.length > 0) {
|
|
507
|
-
pushMarkdownChunk(buffer.join("\n\n"), bufferStart, endLine);
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
for (let i = 0; i < lines.length; i++) {
|
|
512
|
-
const line = lines[i];
|
|
513
|
-
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
514
|
-
|
|
515
|
-
if (headingMatch && sectionLines.length > 0) {
|
|
516
|
-
flush(i);
|
|
517
|
-
sectionLines = [];
|
|
518
|
-
startLine = i + 1;
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
if (headingMatch) {
|
|
522
|
-
const level = headingMatch[1].length;
|
|
523
|
-
headingStack.splice(level - 1);
|
|
524
|
-
headingStack[level - 1] = line;
|
|
525
|
-
currentHeading = line;
|
|
526
|
-
} else {
|
|
527
|
-
sectionLines.push(line);
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
if (sectionLines.length > 0) {
|
|
532
|
-
flush(lines.length);
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
return chunks;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
export function chunkText(content: string, filePath: string): Omit<ChunkInsert, "kb_id">[] {
|
|
539
|
-
const fileType = detectFileType(filePath);
|
|
540
|
-
const paragraphs = content.split(/\n\n+/);
|
|
541
|
-
const chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
542
|
-
let buffer: string[] = [];
|
|
543
|
-
let bufferTokens = 0;
|
|
544
|
-
let lineOffset = 1;
|
|
545
|
-
|
|
546
|
-
function flush(): void {
|
|
547
|
-
const text = buffer.join("\n\n").trim();
|
|
548
|
-
if (text.length < 50) return;
|
|
549
|
-
chunks.push(makeChunk(text, filePath, fileType, lineOffset, lineOffset + text.split("\n").length));
|
|
550
|
-
lineOffset += text.split("\n").length + 1;
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
function pushOversizedParagraph(para: string): void {
|
|
554
|
-
let offset = 0;
|
|
555
|
-
while (offset < para.length) {
|
|
556
|
-
const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
|
|
557
|
-
if (slice.length >= 50) {
|
|
558
|
-
chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
|
|
559
|
-
}
|
|
560
|
-
lineOffset += slice.split("\n").length;
|
|
561
|
-
offset += MAX_TEXT_CHUNK_CHARS;
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
for (const para of paragraphs) {
|
|
566
|
-
const paraTokens = estimateTokens(para);
|
|
567
|
-
if (para.length > MAX_TEXT_CHUNK_CHARS) {
|
|
568
|
-
if (buffer.length > 0) {
|
|
569
|
-
flush();
|
|
570
|
-
buffer = [];
|
|
571
|
-
bufferTokens = 0;
|
|
572
|
-
}
|
|
573
|
-
pushOversizedParagraph(para);
|
|
574
|
-
continue;
|
|
575
|
-
}
|
|
576
|
-
if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
|
|
577
|
-
flush();
|
|
578
|
-
buffer = [];
|
|
579
|
-
bufferTokens = 0;
|
|
580
|
-
}
|
|
581
|
-
buffer.push(para);
|
|
582
|
-
bufferTokens += paraTokens;
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
if (buffer.length > 0) flush();
|
|
586
|
-
return chunks;
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
export async function chunkFile(content: string, filePath: string): Promise<Omit<ChunkInsert, "kb_id">[]> {
|
|
590
|
-
const fileType = detectFileType(filePath);
|
|
591
|
-
let chunks: Omit<ChunkInsert, "kb_id">[] = [];
|
|
592
|
-
|
|
593
|
-
if (fileType === "markdown") {
|
|
594
|
-
chunks = chunkMarkdown(content, filePath);
|
|
595
|
-
} else if (["typescript", "javascript", "python", "go", "rust", "java"].includes(fileType)) {
|
|
596
|
-
try {
|
|
597
|
-
const { chunkWithAST } = await import("./chunkers/code-ast.ts");
|
|
598
|
-
chunks = await chunkWithAST(content, filePath, fileType);
|
|
599
|
-
} catch {
|
|
600
|
-
/* fallback below */
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
if (chunks.length === 0) chunks = chunkText(content, filePath);
|
|
605
|
-
|
|
606
|
-
// Fallback: if file has content but no chunks (too short for splitting), keep as single chunk
|
|
607
|
-
if (chunks.length === 0 && content.trim().length > 10) {
|
|
608
|
-
chunks = [makeChunk(content.trim(), filePath, fileType, 1, content.split("\n").length)];
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
return chunks;
|
|
612
|
-
}
|