pi-knowledge 0.3.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,20 +1,103 @@
1
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
1
+ import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
2
2
  import { dirname } from "node:path";
3
3
 
4
- export function saveVectors(path: string, vectors: Float32Array[]): void {
5
- if (vectors.length === 0) return;
4
+ export interface VectorWriter {
5
+ append(vectors: Float32Array[]): void;
6
+ close(): void;
7
+ }
8
+
9
+ export interface VectorReader {
10
+ count: number;
11
+ dim: number;
12
+ read(index: number): Float32Array | undefined;
13
+ readInto(index: number, target: Float32Array): boolean;
14
+ close(): void;
15
+ }
16
+
17
+ export function openVectorWriter(path: string): VectorWriter {
6
18
  const dir = dirname(path);
7
19
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
8
- const dim = vectors[0].length;
9
- const buffer = Buffer.alloc(8 + vectors.length * dim * 4);
10
- buffer.writeUInt32LE(vectors.length, 0);
11
- buffer.writeUInt32LE(dim, 4);
12
- let offset = 8;
13
- for (const vec of vectors) {
14
- Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength).copy(buffer, offset);
15
- offset += dim * 4;
20
+ const fd = openSync(path, "w");
21
+ let count = 0;
22
+ let dim = 0;
23
+ let closed = false;
24
+ let byteOffset = 8;
25
+ writeSync(fd, Buffer.alloc(8), 0, 8, 0);
26
+
27
+ function writeHeader(): void {
28
+ const header = Buffer.alloc(8);
29
+ header.writeUInt32LE(count, 0);
30
+ header.writeUInt32LE(dim, 4);
31
+ writeSync(fd, header, 0, header.length, 0);
32
+ }
33
+
34
+ return {
35
+ append(vectors: Float32Array[]): void {
36
+ if (closed) throw new Error("Vector writer is already closed");
37
+ for (const vector of vectors) {
38
+ if (dim === 0) dim = vector.length;
39
+ if (vector.length !== dim) {
40
+ throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
41
+ }
42
+ const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
43
+ writeSync(fd, bytes, 0, bytes.length, byteOffset);
44
+ byteOffset += bytes.length;
45
+ count++;
46
+ }
47
+ },
48
+ close(): void {
49
+ if (closed) return;
50
+ writeHeader();
51
+ closeSync(fd);
52
+ closed = true;
53
+ },
54
+ };
55
+ }
56
+
57
+ export function openVectorReader(path: string): VectorReader | undefined {
58
+ if (!existsSync(path)) return undefined;
59
+ const fd = openSync(path, "r");
60
+ let closed = false;
61
+ const header = Buffer.alloc(8);
62
+ const headerBytes = readSync(fd, header, 0, header.length, 0);
63
+ if (headerBytes < header.length) {
64
+ closeSync(fd);
65
+ return undefined;
66
+ }
67
+ const count = header.readUInt32LE(0);
68
+ const dim = header.readUInt32LE(4);
69
+ const vectorBytes = dim * 4;
70
+ const readBuffer = Buffer.alloc(vectorBytes);
71
+
72
+ return {
73
+ count,
74
+ dim,
75
+ read(index: number): Float32Array | undefined {
76
+ const vector = new Float32Array(dim);
77
+ return this.readInto(index, vector) ? vector : undefined;
78
+ },
79
+ readInto(index: number, target: Float32Array): boolean {
80
+ if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
81
+ const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
82
+ if (bytesRead < vectorBytes) return false;
83
+ for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
84
+ return true;
85
+ },
86
+ close(): void {
87
+ if (closed) return;
88
+ closeSync(fd);
89
+ closed = true;
90
+ },
91
+ };
92
+ }
93
+
94
+ export function saveVectors(path: string, vectors: Float32Array[]): void {
95
+ const writer = openVectorWriter(path);
96
+ try {
97
+ writer.append(vectors);
98
+ } finally {
99
+ writer.close();
16
100
  }
17
- writeFileSync(path, buffer);
18
101
  }
19
102
 
20
103
  export function loadVectors(path: string): Float32Array[] {