pi-knowledge 0.3.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/skills/search-docs.md +24 -4
- package/CHANGELOG.md +9 -2
- package/README.md +40 -2
- package/index.ts +200 -14
- package/package.json +1 -1
- package/src/diagnostics/health.ts +84 -14
- package/src/embedding/vectors.ts +95 -12
- package/src/engine.ts +880 -140
- package/src/indexer/chunker.ts +232 -24
- package/src/search/bm25.ts +15 -4
- package/src/search/ranking.ts +4 -0
- package/src/search/vector.ts +60 -0
- package/src/storage/sqlite.ts +224 -9
package/src/embedding/vectors.ts
CHANGED
|
@@ -1,20 +1,103 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync,
|
|
1
|
+
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
|
|
4
|
-
export
|
|
5
|
-
|
|
4
|
+
export interface VectorWriter {
|
|
5
|
+
append(vectors: Float32Array[]): void;
|
|
6
|
+
close(): void;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface VectorReader {
|
|
10
|
+
count: number;
|
|
11
|
+
dim: number;
|
|
12
|
+
read(index: number): Float32Array | undefined;
|
|
13
|
+
readInto(index: number, target: Float32Array): boolean;
|
|
14
|
+
close(): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function openVectorWriter(path: string): VectorWriter {
|
|
6
18
|
const dir = dirname(path);
|
|
7
19
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
8
|
-
const
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
let
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
20
|
+
const fd = openSync(path, "w");
|
|
21
|
+
let count = 0;
|
|
22
|
+
let dim = 0;
|
|
23
|
+
let closed = false;
|
|
24
|
+
let byteOffset = 8;
|
|
25
|
+
writeSync(fd, Buffer.alloc(8), 0, 8, 0);
|
|
26
|
+
|
|
27
|
+
function writeHeader(): void {
|
|
28
|
+
const header = Buffer.alloc(8);
|
|
29
|
+
header.writeUInt32LE(count, 0);
|
|
30
|
+
header.writeUInt32LE(dim, 4);
|
|
31
|
+
writeSync(fd, header, 0, header.length, 0);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return {
|
|
35
|
+
append(vectors: Float32Array[]): void {
|
|
36
|
+
if (closed) throw new Error("Vector writer is already closed");
|
|
37
|
+
for (const vector of vectors) {
|
|
38
|
+
if (dim === 0) dim = vector.length;
|
|
39
|
+
if (vector.length !== dim) {
|
|
40
|
+
throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
|
|
41
|
+
}
|
|
42
|
+
const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
43
|
+
writeSync(fd, bytes, 0, bytes.length, byteOffset);
|
|
44
|
+
byteOffset += bytes.length;
|
|
45
|
+
count++;
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
close(): void {
|
|
49
|
+
if (closed) return;
|
|
50
|
+
writeHeader();
|
|
51
|
+
closeSync(fd);
|
|
52
|
+
closed = true;
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function openVectorReader(path: string): VectorReader | undefined {
|
|
58
|
+
if (!existsSync(path)) return undefined;
|
|
59
|
+
const fd = openSync(path, "r");
|
|
60
|
+
let closed = false;
|
|
61
|
+
const header = Buffer.alloc(8);
|
|
62
|
+
const headerBytes = readSync(fd, header, 0, header.length, 0);
|
|
63
|
+
if (headerBytes < header.length) {
|
|
64
|
+
closeSync(fd);
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
const count = header.readUInt32LE(0);
|
|
68
|
+
const dim = header.readUInt32LE(4);
|
|
69
|
+
const vectorBytes = dim * 4;
|
|
70
|
+
const readBuffer = Buffer.alloc(vectorBytes);
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
count,
|
|
74
|
+
dim,
|
|
75
|
+
read(index: number): Float32Array | undefined {
|
|
76
|
+
const vector = new Float32Array(dim);
|
|
77
|
+
return this.readInto(index, vector) ? vector : undefined;
|
|
78
|
+
},
|
|
79
|
+
readInto(index: number, target: Float32Array): boolean {
|
|
80
|
+
if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
|
|
81
|
+
const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
|
|
82
|
+
if (bytesRead < vectorBytes) return false;
|
|
83
|
+
for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
|
|
84
|
+
return true;
|
|
85
|
+
},
|
|
86
|
+
close(): void {
|
|
87
|
+
if (closed) return;
|
|
88
|
+
closeSync(fd);
|
|
89
|
+
closed = true;
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function saveVectors(path: string, vectors: Float32Array[]): void {
|
|
95
|
+
const writer = openVectorWriter(path);
|
|
96
|
+
try {
|
|
97
|
+
writer.append(vectors);
|
|
98
|
+
} finally {
|
|
99
|
+
writer.close();
|
|
16
100
|
}
|
|
17
|
-
writeFileSync(path, buffer);
|
|
18
101
|
}
|
|
19
102
|
|
|
20
103
|
export function loadVectors(path: string): Float32Array[] {
|