pi-knowledge 0.3.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { createHash } from "node:crypto";
2
- import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
2
+ import { closeSync, type Dirent, existsSync, openSync, readdirSync, readFileSync, readSync, statSync } from "node:fs";
3
3
  import { extname, join, relative, sep } from "node:path";
4
4
  import ignore from "ignore";
5
5
  import type { ChunkInsert } from "../storage/sqlite.ts";
@@ -54,7 +54,7 @@ const BINARY_EXTENSIONS = new Set([
54
54
  ".lib",
55
55
  ]);
56
56
 
57
- const DEFAULT_IGNORE = [
57
+ const DEFAULT_SUGGESTED_EXCLUDE = [
58
58
  "node_modules",
59
59
  ".git",
60
60
  "dist",
@@ -67,7 +67,13 @@ const DEFAULT_IGNORE = [
67
67
  ".next",
68
68
  ".cache",
69
69
  ".playwright",
70
+ ".browser",
71
+ ".browsers",
70
72
  "__pycache__",
73
+ "*.app",
74
+ "*.asar",
75
+ "*.asar.unpacked",
76
+ "*.pak",
71
77
  ".env",
72
78
  ".env.*",
73
79
  "*.pem",
@@ -84,19 +90,28 @@ const DEFAULT_IGNORE = [
84
90
  "docs/*evaluation-report*.md",
85
91
  "docs/*knowledge-base*report*.md",
86
92
  "*knowledge-base-evaluation-report*.md",
87
- "setting*.json",
88
- "appsettings*.json",
93
+ "*knowledge*.jsonl",
89
94
  "*.min.js",
90
95
  "*.min.css",
91
96
  "*.map",
92
97
  "*.lock",
93
98
  "package-lock.json",
99
+ "playwright-report",
100
+ "test-results",
101
+ "browser-cache",
102
+ "ms-playwright",
94
103
  ".DS_Store",
95
104
  "Thumbs.db",
96
105
  "*.pyc",
97
106
  "*.class",
98
107
  ];
99
108
 
109
+ export interface ScanOptions {
110
+ includeSuggestedText?: boolean;
111
+ includePaths?: string[];
112
+ excludePaths?: string[];
113
+ }
114
+
100
115
  export interface ScannedFile {
101
116
  path: string; // absolute path
102
117
  relPath: string; // relative to root
@@ -104,6 +119,59 @@ export interface ScannedFile {
104
119
  fileType: string;
105
120
  }
106
121
 
122
+ export interface ScannableFile {
123
+ path: string; // absolute path
124
+ relPath: string; // relative to root
125
+ fileType: string;
126
+ size: number;
127
+ }
128
+
129
+ export interface SkippedScanEntry {
130
+ path: string;
131
+ reason: "suggested_excluded" | "oversized" | "binary" | "unreadable" | "inaccessible";
132
+ size?: number;
133
+ }
134
+
135
+ export interface ScanResult {
136
+ files: ScannedFile[];
137
+ skipped: {
138
+ total: number;
139
+ by_reason: Record<SkippedScanEntry["reason"], number>;
140
+ samples: SkippedScanEntry[];
141
+ };
142
+ }
143
+
144
+ const MAX_SKIPPED_SAMPLES = 25;
145
+
146
+ export function createSkippedScanStats(): ScanResult["skipped"] {
147
+ return {
148
+ total: 0,
149
+ by_reason: {
150
+ suggested_excluded: 0,
151
+ oversized: 0,
152
+ binary: 0,
153
+ unreadable: 0,
154
+ inaccessible: 0,
155
+ },
156
+ samples: [],
157
+ };
158
+ }
159
+
160
+ function addSkipped(skipped: ScanResult["skipped"], entry: SkippedScanEntry): void {
161
+ skipped.total++;
162
+ skipped.by_reason[entry.reason]++;
163
+ if (skipped.samples.length < MAX_SKIPPED_SAMPLES) skipped.samples.push(entry);
164
+ }
165
+
166
+ export function summarizeSkippedScan(skipped: ScanResult["skipped"]): string {
167
+ return (
168
+ Object.entries(skipped.by_reason)
169
+ .filter(([, count]) => count > 0)
170
+ .map(([reason, count]) => `${reason}: ${count}`)
171
+ .join(", ") || "none"
172
+ );
173
+ }
174
+
107
175
  function detectFileType(filePath: string): string {
108
176
  const ext = extname(filePath).toLowerCase();
109
177
  const map: Record<string, string> = {
@@ -143,59 +211,162 @@ function detectFileType(filePath: string): string {
143
211
 
144
212
  function isBinaryFile(filePath: string): boolean {
145
213
  if (BINARY_EXTENSIONS.has(extname(filePath).toLowerCase())) return true;
214
+ let fd: number | undefined;
146
215
  try {
147
- const fd = readFileSync(filePath, { flag: "r" });
148
- const sample = fd.subarray(0, 512);
149
- return sample.includes(0x00);
216
+ fd = openSync(filePath, "r");
217
+ const sample = Buffer.alloc(512);
218
+ const bytesRead = readSync(fd, sample, 0, sample.length, 0);
219
+ if (bytesRead === 0) return false;
220
+ return sample.subarray(0, bytesRead).includes(0x00);
150
221
  } catch {
151
222
  return true;
223
+ } finally {
224
+ if (fd !== undefined) closeSync(fd);
152
225
  }
153
226
  }
154
227
 
155
- export function walkDir(dirPath: string): ScannedFile[] {
228
+ function buildIgnoreMatcher(dirPath: string): ReturnType<typeof ignore> {
156
229
  const ig = ignore();
157
- ig.add(DEFAULT_IGNORE);
230
+ ig.add(DEFAULT_SUGGESTED_EXCLUDE);
158
231
 
159
232
  const gitignorePath = join(dirPath, ".gitignore");
160
233
  if (existsSync(gitignorePath)) {
161
234
  ig.add(readFileSync(gitignorePath, "utf-8"));
162
235
  }
163
236
 
164
- const results: ScannedFile[] = [];
237
+ return ig;
238
+ }
239
+
240
+ function normalizeScanPath(path: string): string {
241
+ return path.replaceAll("\\", "/").replace(/^\/+/, "").replace(/\/+$/, "");
242
+ }
243
+
244
+ function normalizeScanPaths(paths: string[] | undefined): string[] {
245
+ return (paths ?? []).map(normalizeScanPath).filter(Boolean);
246
+ }
165
247
 
166
- function walk(dir: string): void {
248
+ function pathMatches(relPath: string, patterns: string[]): boolean {
249
+ const normalized = normalizeScanPath(relPath);
250
+ return patterns.some((pattern) => normalized === pattern || normalized.startsWith(`${pattern}/`));
251
+ }
252
+
253
+ function directoryCouldContainIncludedPath(relPath: string, includePaths: string[]): boolean {
254
+ const normalized = normalizeScanPath(relPath);
255
+ return includePaths.some((includePath) => includePath.startsWith(`${normalized}/`));
256
+ }
257
+
258
+ export function* iterateScannableFiles(
259
+ dirPath: string,
260
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
261
+ options: ScanOptions = {},
262
+ ): Generator<ScannableFile> {
263
+ const ig = buildIgnoreMatcher(dirPath);
264
+ const includePaths = normalizeScanPaths(options.includePaths);
265
+ const excludePaths = normalizeScanPaths(options.excludePaths);
266
+ const includeSuggestedText = options.includeSuggestedText === true;
267
+
268
+ function* walk(dir: string): Generator<ScannableFile> {
167
269
  let entries: Dirent[];
168
270
  try {
169
271
  entries = readdirSync(dir, { withFileTypes: true });
170
272
  } catch {
273
+ addSkipped(skipped, { path: relative(dirPath, dir).split(sep).join("/") || ".", reason: "inaccessible" });
171
274
  return;
172
275
  }
173
276
  for (const entry of entries) {
174
277
  const fullPath = join(dir, entry.name);
175
278
  const relPath = relative(dirPath, fullPath).split(sep).join("/");
279
+ const explicitlyIncluded = pathMatches(relPath, includePaths);
176
280
 
177
- if (ig.ignores(relPath)) continue;
281
+ if (pathMatches(relPath, excludePaths)) {
282
+ addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
283
+ continue;
284
+ }
285
+
286
+ if (ig.ignores(relPath) && !includeSuggestedText && !explicitlyIncluded) {
287
+ if (entry.isDirectory() && directoryCouldContainIncludedPath(relPath, includePaths)) {
288
+ yield* walk(fullPath);
289
+ continue;
290
+ }
291
+ addSkipped(skipped, { path: relPath, reason: "suggested_excluded" });
292
+ continue;
293
+ }
178
294
 
179
295
  if (entry.isDirectory()) {
180
- if (ig.ignores(`${relPath}/`)) continue;
181
- walk(fullPath);
296
+ if (ig.ignores(`${relPath}/`) && !includeSuggestedText && !explicitlyIncluded) {
297
+ if (directoryCouldContainIncludedPath(relPath, includePaths)) {
298
+ yield* walk(fullPath);
299
+ continue;
300
+ }
301
+ addSkipped(skipped, { path: `${relPath}/`, reason: "suggested_excluded" });
302
+ continue;
303
+ }
304
+ yield* walk(fullPath);
182
305
  } else if (entry.isFile()) {
183
- const stat = statSync(fullPath);
184
- if (stat.size > MAX_FILE_SIZE) continue;
185
- if (isBinaryFile(fullPath)) continue;
186
-
306
+ let size = 0;
187
307
  try {
188
- const content = readFileSync(fullPath, "utf-8");
189
- results.push({ path: fullPath, relPath, content, fileType: detectFileType(fullPath) });
308
+ size = statSync(fullPath).size;
190
309
  } catch {
191
- // skip unreadable files
310
+ addSkipped(skipped, { path: relPath, reason: "unreadable" });
311
+ continue;
312
+ }
313
+ if (size > MAX_FILE_SIZE) {
314
+ addSkipped(skipped, { path: relPath, reason: "oversized", size });
315
+ continue;
316
+ }
317
+ if (isBinaryFile(fullPath)) {
318
+ addSkipped(skipped, { path: relPath, reason: "binary", size });
319
+ continue;
192
320
  }
321
+
322
+ yield { path: fullPath, relPath, fileType: detectFileType(fullPath), size };
193
323
  }
194
324
  }
195
325
  }
196
326
 
197
- walk(dirPath);
198
- return results;
327
+ yield* walk(dirPath);
328
+ }
329
+
330
+ export function* iterateScannedFiles(
331
+ dirPath: string,
332
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
333
+ options: ScanOptions = {},
334
+ ): Generator<ScannedFile> {
335
+ for (const file of iterateScannableFiles(dirPath, skipped, options)) {
336
+ try {
337
+ const content = readFileSync(file.path, "utf-8");
338
+ yield { ...file, content };
339
+ } catch {
340
+ addSkipped(skipped, { path: file.relPath, reason: "unreadable", size: file.size });
341
+ }
342
+ }
343
+ }
344
+
345
+ export async function iterateScannedFilesAsync(
346
+ dirPath: string,
347
+ onFile: (file: ScannedFile) => Promise<void> | void,
348
+ skipped: ScanResult["skipped"] = createSkippedScanStats(),
349
+ options: ScanOptions = {},
350
+ ): Promise<ScanResult["skipped"]> {
351
+ for (const file of iterateScannedFiles(dirPath, skipped, options)) {
352
+ await onFile(file);
353
+ }
354
+ return skipped;
355
+ }
356
+
357
+ export function walkDir(dirPath: string, options: ScanOptions = {}): ScannedFile[] {
358
+ return walkDirDetailed(dirPath, options).files;
359
+ }
360
+
361
+ export function walkDirDetailed(dirPath: string, options: ScanOptions = {}): ScanResult {
362
+ const results: ScannedFile[] = [];
363
+ const skipped = createSkippedScanStats();
364
+ for (const file of iterateScannedFiles(dirPath, skipped, options)) results.push(file);
365
+ return { files: results, skipped };
366
+ }
367
+
368
+ export function isReadableTextFile(filePath: string): boolean {
369
+ return !isBinaryFile(filePath);
199
370
  }
200
371
 
201
372
  // --- Chunking ---
@@ -206,6 +377,7 @@ function estimateTokens(text: string): number {
206
377
 
207
378
  const MARKDOWN_TARGET_TOKENS = 450;
208
379
  const TEXT_TARGET_TOKENS = 550;
380
+ const MAX_TEXT_CHUNK_CHARS = 6_000;
209
381
 
210
382
  export function preTokenizeForFTS(content: string): string {
211
383
  return content
@@ -222,6 +394,21 @@ export function contentHash(content: string): string {
222
394
  return createHash("sha256").update(content).digest("hex");
223
395
  }
224
396
 
397
+ export function chunkIdentityHash(opts: {
398
+ content: string;
399
+ filePath: string;
400
+ fileType: string;
401
+ startLine: number;
402
+ endLine: number;
403
+ metadataJson: string;
404
+ }): string {
405
+ return contentHash(
406
+ [opts.filePath, opts.fileType, String(opts.startLine), String(opts.endLine), opts.metadataJson, opts.content].join(
407
+ "\0",
408
+ ),
409
+ );
410
+ }
411
+
225
412
  function normalizeHeading(heading: string): string {
226
413
  return heading.replace(/^#{1,6}\s+/, "").trim();
227
414
  }
@@ -263,7 +450,7 @@ function makeChunk(
263
450
  ): Omit<ChunkInsert, "kb_id"> {
264
451
  const metadata_json = JSON.stringify(metadata);
265
452
  const chunk = {
266
- content_hash: contentHash(content),
453
+ content_hash: chunkIdentityHash({ content, filePath, fileType, startLine, endLine, metadataJson: metadata_json }),
267
454
  content,
268
455
  content_tokenized: "",
269
456
  file_path: filePath,
@@ -363,8 +550,29 @@ export function chunkText(content: string, filePath: string): Omit<ChunkInsert,
363
550
  lineOffset += text.split("\n").length + 1;
364
551
  }
365
552
 
553
+ function pushOversizedParagraph(para: string): void {
554
+ let offset = 0;
555
+ while (offset < para.length) {
556
+ const slice = para.slice(offset, offset + MAX_TEXT_CHUNK_CHARS).trim();
557
+ if (slice.length >= 50) {
558
+ chunks.push(makeChunk(slice, filePath, fileType, lineOffset, lineOffset + slice.split("\n").length));
559
+ }
560
+ lineOffset += slice.split("\n").length;
561
+ offset += MAX_TEXT_CHUNK_CHARS;
562
+ }
563
+ }
564
+
366
565
  for (const para of paragraphs) {
367
566
  const paraTokens = estimateTokens(para);
567
+ if (para.length > MAX_TEXT_CHUNK_CHARS) {
568
+ if (buffer.length > 0) {
569
+ flush();
570
+ buffer = [];
571
+ bufferTokens = 0;
572
+ }
573
+ pushOversizedParagraph(para);
574
+ continue;
575
+ }
368
576
  if (bufferTokens + paraTokens > TEXT_TARGET_TOKENS && buffer.length > 0) {
369
577
  flush();
370
578
  buffer = [];
@@ -12,6 +12,10 @@ function prepareFtsTerms(query: string): string[] {
12
12
  .filter((t) => t.length > 0);
13
13
  }
14
14
 
15
+ function quoteFtsTerm(term: string): string {
16
+ return `"${term.replace(/"/g, '""')}"`;
17
+ }
18
+
15
19
  function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?: string): BM25Result[] {
16
20
  if (kbId) {
17
21
  return db
@@ -31,13 +35,20 @@ function runSearch(db: Database.Database, ftsQuery: string, limit: number, kbId?
31
35
  .all(ftsQuery, limit) as BM25Result[];
32
36
  }
33
37
 
34
- export function searchBM25(db: Database.Database, query: string, limit = 50, kbId?: string): BM25Result[] {
38
+ export function searchBM25(
39
+ db: Database.Database,
40
+ query: string,
41
+ limit = 50,
42
+ kbId?: string,
43
+ options: { allowOrFallback?: boolean } = {},
44
+ ): BM25Result[] {
35
45
  const terms = prepareFtsTerms(query);
36
46
  if (terms.length === 0) return [];
37
47
  try {
38
- const strict = runSearch(db, terms.join(" AND "), limit, kbId);
39
- if (strict.length > 0 || terms.length === 1) return strict;
40
- return runSearch(db, terms.join(" OR "), limit, kbId);
48
+ const quotedTerms = terms.map(quoteFtsTerm);
49
+ const strict = runSearch(db, quotedTerms.join(" AND "), limit, kbId);
50
+ if (strict.length > 0 || terms.length === 1 || options.allowOrFallback === false) return strict;
51
+ return runSearch(db, quotedTerms.join(" OR "), limit, kbId);
41
52
  } catch {
42
53
  return [];
43
54
  }
@@ -108,6 +108,10 @@ export function queryCoverage(text: string, queryTokens: Set<string>): number {
108
108
  return matched / signals.size;
109
109
  }
110
110
 
111
+ export function hasAnyLexicalEvidence(text: string, queryTokens: Set<string>): boolean {
112
+ return queryCoverage(text, queryTokens) > 0;
113
+ }
114
+
111
115
  export function pathTokenBoost(filePath: string, queryTokens: Set<string>): number {
112
116
  const pathTokens = signalTokens(tokenizeForSearch(filePath.replace(/[./_-]/g, " ")));
113
117
  const signals = signalTokens(queryTokens);
@@ -1,14 +1,42 @@
1
+ import { openVectorReader } from "../embedding/vectors.ts";
2
+
1
3
  export interface VectorResult {
2
4
  chunkId: string;
3
5
  score: number;
4
6
  }
5
7
 
8
+ export interface VectorFileSearchResult {
9
+ results: VectorResult[];
10
+ vectorsByChunkId: Map<string, Float32Array>;
11
+ }
12
+
13
+ type ChunkIdSource = Iterable<string | { id: string }>;
14
+
6
15
  function cosine(a: Float32Array, b: Float32Array): number {
7
16
  let dot = 0;
8
17
  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
9
18
  return dot;
10
19
  }
11
20
 
21
+ function keepTopK(
22
+ top: Array<VectorResult & { vector: Float32Array }>,
23
+ item: VectorResult,
24
+ vector: Float32Array,
25
+ limit: number,
26
+ ): void {
27
+ if (limit <= 0) return;
28
+ if (top.length < limit) {
29
+ const stored = { ...item, vector: new Float32Array(vector) };
30
+ top.push(stored);
31
+ top.sort((a, b) => a.score - b.score);
32
+ return;
33
+ }
34
+ if (item.score <= top[0].score) return;
35
+ const stored = { ...item, vector: new Float32Array(vector) };
36
+ top[0] = stored;
37
+ top.sort((a, b) => a.score - b.score);
38
+ }
39
+
12
40
  export function searchVector(
13
41
  queryVec: Float32Array,
14
42
  vectors: Float32Array[],
@@ -19,3 +47,35 @@ export function searchVector(
19
47
  scores.sort((a, b) => b.score - a.score);
20
48
  return scores.slice(0, limit);
21
49
  }
50
+
51
+ export function searchVectorFile(
52
+ queryVec: Float32Array,
53
+ vectorPath: string,
54
+ chunkIds: ChunkIdSource,
55
+ limit = 50,
56
+ ): VectorFileSearchResult {
57
+ const reader = openVectorReader(vectorPath);
58
+ if (!reader) return { results: [], vectorsByChunkId: new Map() };
59
+ try {
60
+ if (reader.dim !== queryVec.length) return { results: [], vectorsByChunkId: new Map() };
61
+ const scratch = new Float32Array(reader.dim);
62
+ const top: Array<VectorResult & { vector: Float32Array }> = [];
63
+ let i = 0;
64
+ for (const chunkIdRef of chunkIds) {
65
+ if (i >= reader.count) break;
66
+ const vectorIndex = i;
67
+ i++;
68
+ if (!reader.readInto(vectorIndex, scratch)) continue;
69
+ const chunkId = typeof chunkIdRef === "string" ? chunkIdRef : chunkIdRef.id;
70
+ keepTopK(top, { chunkId, score: cosine(queryVec, scratch) }, scratch, limit);
71
+ }
72
+ const sorted = top.sort((a, b) => b.score - a.score);
73
+ const vectorsByChunkId = new Map(sorted.map((item) => [item.chunkId, item.vector]));
74
+ return {
75
+ results: sorted.map(({ chunkId, score }) => ({ chunkId, score })),
76
+ vectorsByChunkId,
77
+ };
78
+ } finally {
79
+ reader.close();
80
+ }
81
+ }