pi-knowledge 0.4.1 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +29 -8
  3. package/dist/index.d.ts +38 -0
  4. package/dist/index.js +435 -0
  5. package/dist/src/diagnostics/health.d.ts +19 -0
  6. package/dist/src/diagnostics/health.js +110 -0
  7. package/dist/src/embedding/provider.d.ts +5 -0
  8. package/dist/src/embedding/provider.js +99 -0
  9. package/dist/src/embedding/vectors.d.ts +15 -0
  10. package/dist/src/embedding/vectors.js +111 -0
  11. package/dist/src/engine.d.ts +90 -0
  12. package/dist/src/engine.js +1370 -0
  13. package/dist/src/indexer/chunker.d.ts +53 -0
  14. package/dist/src/indexer/chunker.js +498 -0
  15. package/dist/src/indexer/chunkers/code-ast.d.ts +3 -0
  16. package/dist/src/indexer/chunkers/code-ast.js +179 -0
  17. package/dist/src/model-worker-client.d.ts +10 -0
  18. package/dist/src/model-worker-client.js +138 -0
  19. package/dist/src/model-worker.d.ts +1 -0
  20. package/dist/src/model-worker.js +61 -0
  21. package/dist/src/search/bm25.d.ts +8 -0
  22. package/dist/src/search/bm25.js +38 -0
  23. package/dist/src/search/fusion.d.ts +19 -0
  24. package/dist/src/search/fusion.js +34 -0
  25. package/dist/src/search/query.d.ts +5 -0
  26. package/dist/src/search/query.js +57 -0
  27. package/dist/src/search/ranking.d.ts +27 -0
  28. package/dist/src/search/ranking.js +180 -0
  29. package/dist/src/search/reranker.d.ts +10 -0
  30. package/dist/src/search/reranker.js +66 -0
  31. package/dist/src/search/vector.d.ts +14 -0
  32. package/dist/src/search/vector.js +58 -0
  33. package/dist/src/storage/sqlite.d.ts +99 -0
  34. package/dist/src/storage/sqlite.js +330 -0
  35. package/dist/src/watcher/file-watcher.d.ts +4 -0
  36. package/dist/src/watcher/file-watcher.js +108 -0
  37. package/extension.js +9 -0
  38. package/package.json +20 -6
  39. package/index.ts +0 -500
  40. package/src/diagnostics/health.ts +0 -145
  41. package/src/embedding/provider.ts +0 -106
  42. package/src/embedding/vectors.ts +0 -119
  43. package/src/engine.ts +0 -1592
  44. package/src/indexer/chunker.ts +0 -612
  45. package/src/indexer/chunkers/code-ast.ts +0 -209
  46. package/src/model-worker-client.ts +0 -146
  47. package/src/model-worker.ts +0 -88
  48. package/src/search/bm25.ts +0 -55
  49. package/src/search/fusion.ts +0 -48
  50. package/src/search/query.ts +0 -59
  51. package/src/search/ranking.ts +0 -201
  52. package/src/search/reranker.ts +0 -74
  53. package/src/search/vector.ts +0 -81
  54. package/src/storage/sqlite.ts +0 -489
  55. package/src/watcher/file-watcher.ts +0 -116
@@ -1,106 +0,0 @@
1
- import { embedInModelWorker } from "../model-worker-client.ts";
2
-
3
- let disposeTimer: ReturnType<typeof setTimeout> | null = null;
4
- let disposePromise: Promise<void> | null = null;
5
- let activeRuns = 0;
6
- let disposeRequested = false;
7
- const idleWaiters: Array<() => void> = [];
8
-
9
- const IDLE_TIMEOUT_MS = Number(process.env.PI_KNOWLEDGE_EMBEDDING_IDLE_MS ?? 30_000);
10
- const EMBEDDING_CONFIG = process.env.PI_KNOWLEDGE_EMBEDDING ?? "local:multilingual-e5-small";
11
- const ENABLE_NATIVE_IDLE_DISPOSE = process.env.PI_KNOWLEDGE_ENABLE_NATIVE_IDLE_DISPOSE === "true";
12
-
13
- function clearIdleTimer(): void {
14
- if (disposeTimer) clearTimeout(disposeTimer);
15
- disposeTimer = null;
16
- }
17
-
18
- function scheduleIdleDispose(): void {
19
- if (activeRuns > 0 || disposeRequested) return;
20
- clearIdleTimer();
21
- if (!ENABLE_NATIVE_IDLE_DISPOSE) return;
22
- disposeTimer = setTimeout(() => dispose(), IDLE_TIMEOUT_MS);
23
- }
24
-
25
- function beginRun(): void {
26
- activeRuns++;
27
- clearIdleTimer();
28
- }
29
-
30
- function endRun(): void {
31
- activeRuns--;
32
- if (activeRuns > 0) return;
33
- for (const resolve of idleWaiters.splice(0)) resolve();
34
- if (!disposeRequested) scheduleIdleDispose();
35
- }
36
-
37
- function waitForNoActiveRuns(): Promise<void> {
38
- if (activeRuns === 0) return Promise.resolve();
39
- return new Promise((resolve) => idleWaiters.push(resolve));
40
- }
41
-
42
- export async function dispose(): Promise<void> {
43
- clearIdleTimer();
44
- if (disposePromise) return disposePromise;
45
- disposeRequested = true;
46
- await waitForNoActiveRuns();
47
- disposePromise = Promise.resolve().finally(() => {
48
- disposePromise = null;
49
- disposeRequested = false;
50
- });
51
- return disposePromise;
52
- }
53
-
54
- export async function prepareForShutdown(): Promise<void> {
55
- clearIdleTimer();
56
- await waitForNoActiveRuns();
57
- }
58
-
59
- async function embedViaAPI(texts: string[], prefix: "query" | "passage"): Promise<Float32Array[]> {
60
- const [provider, model] = EMBEDDING_CONFIG.split(":");
61
- const prefixedTexts = texts.map((t) => `${prefix}: ${t}`);
62
-
63
- if (provider === "openai") {
64
- const apiKey = process.env.OPENAI_API_KEY;
65
- if (!apiKey) throw new Error("OPENAI_API_KEY required for openai embedding");
66
- const res = await fetch("https://api.openai.com/v1/embeddings", {
67
- method: "POST",
68
- headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
69
- body: JSON.stringify({ input: prefixedTexts, model: model || "text-embedding-3-small" }),
70
- });
71
- if (!res.ok) throw new Error(`OpenAI API error: ${res.status}`);
72
- const data = (await res.json()) as { data: Array<{ embedding: number[] }> };
73
- return data.data.map((d) => new Float32Array(d.embedding));
74
- }
75
-
76
- throw new Error(`Unsupported embedding provider: ${provider}`);
77
- }
78
-
79
- export async function embedTexts(
80
- texts: string[],
81
- prefix: "query" | "passage",
82
- signal?: AbortSignal,
83
- ): Promise<Float32Array[]> {
84
- if (!EMBEDDING_CONFIG.startsWith("local")) {
85
- try {
86
- return await embedViaAPI(texts, prefix);
87
- } catch {
88
- /* fallback to local */
89
- }
90
- }
91
- beginRun();
92
- try {
93
- return await embedInModelWorker(texts, prefix, signal);
94
- } finally {
95
- endRun();
96
- }
97
- }
98
-
99
- export async function embedQuery(text: string): Promise<Float32Array> {
100
- const [vec] = await embedTexts([text], "query");
101
- return vec;
102
- }
103
-
104
- export async function embedDocuments(texts: string[], signal?: AbortSignal): Promise<Float32Array[]> {
105
- return embedTexts(texts, "passage", signal);
106
- }
@@ -1,119 +0,0 @@
1
- import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, writeSync } from "node:fs";
2
- import { dirname } from "node:path";
3
-
4
- export interface VectorWriter {
5
- append(vectors: Float32Array[]): void;
6
- close(): void;
7
- }
8
-
9
- export interface VectorReader {
10
- count: number;
11
- dim: number;
12
- read(index: number): Float32Array | undefined;
13
- readInto(index: number, target: Float32Array): boolean;
14
- close(): void;
15
- }
16
-
17
- export function openVectorWriter(path: string): VectorWriter {
18
- const dir = dirname(path);
19
- if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
20
- const fd = openSync(path, "w");
21
- let count = 0;
22
- let dim = 0;
23
- let closed = false;
24
- let byteOffset = 8;
25
- writeSync(fd, Buffer.alloc(8), 0, 8, 0);
26
-
27
- function writeHeader(): void {
28
- const header = Buffer.alloc(8);
29
- header.writeUInt32LE(count, 0);
30
- header.writeUInt32LE(dim, 4);
31
- writeSync(fd, header, 0, header.length, 0);
32
- }
33
-
34
- return {
35
- append(vectors: Float32Array[]): void {
36
- if (closed) throw new Error("Vector writer is already closed");
37
- for (const vector of vectors) {
38
- if (dim === 0) dim = vector.length;
39
- if (vector.length !== dim) {
40
- throw new Error(`Vector dimension mismatch: expected ${dim}, got ${vector.length}`);
41
- }
42
- const bytes = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
43
- writeSync(fd, bytes, 0, bytes.length, byteOffset);
44
- byteOffset += bytes.length;
45
- count++;
46
- }
47
- },
48
- close(): void {
49
- if (closed) return;
50
- writeHeader();
51
- closeSync(fd);
52
- closed = true;
53
- },
54
- };
55
- }
56
-
57
- export function openVectorReader(path: string): VectorReader | undefined {
58
- if (!existsSync(path)) return undefined;
59
- const fd = openSync(path, "r");
60
- let closed = false;
61
- const header = Buffer.alloc(8);
62
- const headerBytes = readSync(fd, header, 0, header.length, 0);
63
- if (headerBytes < header.length) {
64
- closeSync(fd);
65
- return undefined;
66
- }
67
- const count = header.readUInt32LE(0);
68
- const dim = header.readUInt32LE(4);
69
- const vectorBytes = dim * 4;
70
- const readBuffer = Buffer.alloc(vectorBytes);
71
-
72
- return {
73
- count,
74
- dim,
75
- read(index: number): Float32Array | undefined {
76
- const vector = new Float32Array(dim);
77
- return this.readInto(index, vector) ? vector : undefined;
78
- },
79
- readInto(index: number, target: Float32Array): boolean {
80
- if (closed || index < 0 || index >= count || dim === 0 || target.length !== dim) return false;
81
- const bytesRead = readSync(fd, readBuffer, 0, vectorBytes, 8 + index * vectorBytes);
82
- if (bytesRead < vectorBytes) return false;
83
- for (let i = 0; i < dim; i++) target[i] = readBuffer.readFloatLE(i * 4);
84
- return true;
85
- },
86
- close(): void {
87
- if (closed) return;
88
- closeSync(fd);
89
- closed = true;
90
- },
91
- };
92
- }
93
-
94
- export function saveVectors(path: string, vectors: Float32Array[]): void {
95
- const writer = openVectorWriter(path);
96
- try {
97
- writer.append(vectors);
98
- } finally {
99
- writer.close();
100
- }
101
- }
102
-
103
- export function loadVectors(path: string): Float32Array[] {
104
- if (!existsSync(path)) return [];
105
- const buffer = readFileSync(path);
106
- if (buffer.length < 8) return [];
107
- const count = buffer.readUInt32LE(0);
108
- const dim = buffer.readUInt32LE(4);
109
- const vectors: Float32Array[] = [];
110
- for (let i = 0; i < count; i++) {
111
- const offset = 8 + i * dim * 4;
112
- const vec = new Float32Array(dim);
113
- for (let j = 0; j < dim; j++) {
114
- vec[j] = buffer.readFloatLE(offset + j * 4);
115
- }
116
- vectors.push(vec);
117
- }
118
- return vectors;
119
- }