pi-knowledge 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +1 -1
- package/dist/src/model-worker-client.js +235 -76
- package/dist/src/model-worker.js +94 -9
- package/dist/src/storage/sqlite.d.ts +1 -0
- package/dist/src/storage/sqlite.js +23 -5
- package/docs/configuration.md +4 -2
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.5.3] - 2026-07-23
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Added a Windows-safe model-worker transport fallback so OMP hosts without `child_process.fork().send()` can still build local embedding knowledge bases through a stdin/stdout JSONL worker protocol.
|
|
7
|
+
- Validated the isolated worker Node 22+ executable and worker file before local model startup, with clearer `PI_KNOWLEDGE_NODE_PATH` diagnostics.
|
|
8
|
+
- Hardened OMP/Windows storage-path detection for `omp.exe` hosts and Windows-style home path casing.
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Documented Windows OMP local embedding startup requirements, transport fallback behavior, and troubleshooting guidance.
|
|
12
|
+
|
|
3
13
|
## [0.5.2] - 2026-07-19
|
|
4
14
|
|
|
5
15
|
### Fixed
|
package/README.md
CHANGED
|
@@ -184,7 +184,7 @@ Full configuration details are in [docs/configuration.md](docs/configuration.md)
|
|
|
184
184
|
|
|
185
185
|
Default storage is `~/.pi/knowledge` for Pi and `~/.omp/knowledge` for OMP. Explicit overrides are available with `PI_KNOWLEDGE_DIR` and `OMP_KNOWLEDGE_DIR`. Under the default home OMP root, existing legacy `~/.pi/knowledge` data remains visible when `~/.omp/knowledge` has not been created yet.
|
|
186
186
|
|
|
187
|
-
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated model-worker startup, and idempotent shutdown. Compatibility-sensitive releases should validate both Pi and OMP install/runtime flows.
|
|
187
|
+
OMP compatibility covers path resolution, packaged entry loading, native SQLite dependency resolution, isolated model-worker startup, Windows-safe worker transport fallback, and idempotent shutdown. Local embeddings require Node 22+ for the isolated worker; set `PI_KNOWLEDGE_NODE_PATH` to `node.exe` if OMP runs from a non-Node packaged host. Compatibility-sensitive releases should validate both Pi and OMP install/runtime flows.
|
|
188
188
|
|
|
189
189
|
## Large Project Indexing
|
|
190
190
|
|
|
@@ -1,29 +1,72 @@
|
|
|
1
|
-
import { fork } from "node:child_process";
|
|
1
|
+
import { execFileSync, fork, spawn } from "node:child_process";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
2
3
|
import { basename } from "node:path";
|
|
4
|
+
import { StringDecoder } from "node:string_decoder";
|
|
3
5
|
import { fileURLToPath } from "node:url";
|
|
4
6
|
let worker = null;
|
|
5
7
|
let nextRequestId = 1;
|
|
6
|
-
|
|
8
|
+
let resolvedNodeExecPath = null;
|
|
7
9
|
const WORKER_STDERR_TAIL_CHARS = 4_000;
|
|
8
|
-
|
|
9
|
-
for (const request of pending.values()) {
|
|
10
|
-
request.reject(error);
|
|
11
|
-
}
|
|
12
|
-
pending.clear();
|
|
13
|
-
}
|
|
10
|
+
const MIN_NODE_MAJOR = 22;
|
|
14
11
|
function getWorkerPath() {
|
|
15
12
|
const workerFile = fileURLToPath(import.meta.url).endsWith(".js") ? "model-worker.js" : "model-worker.ts";
|
|
16
|
-
|
|
13
|
+
const workerPath = fileURLToPath(new URL(`./${workerFile}`, import.meta.url));
|
|
14
|
+
if (!existsSync(workerPath)) {
|
|
15
|
+
throw new Error(`Model worker file not found at ${workerPath}. Rebuild or reinstall pi-knowledge so the worker is packaged beside model-worker-client.`);
|
|
16
|
+
}
|
|
17
|
+
return workerPath;
|
|
17
18
|
}
|
|
18
19
|
function getWorkerExecArgv() {
|
|
19
20
|
return fileURLToPath(import.meta.url).endsWith(".js") ? [] : ["--experimental-strip-types"];
|
|
20
21
|
}
|
|
22
|
+
function parseNodeMajor(version) {
|
|
23
|
+
const match = /^v?(?<major>\d+)\./.exec(version.trim());
|
|
24
|
+
if (!match?.groups?.major)
|
|
25
|
+
return null;
|
|
26
|
+
const major = Number(match.groups.major);
|
|
27
|
+
return Number.isInteger(major) ? major : null;
|
|
28
|
+
}
|
|
29
|
+
function validateNodeExecPath(candidate) {
|
|
30
|
+
const version = execFileSync(candidate, ["--version"], {
|
|
31
|
+
encoding: "utf8",
|
|
32
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
33
|
+
timeout: 5_000,
|
|
34
|
+
windowsHide: true,
|
|
35
|
+
}).trim();
|
|
36
|
+
const major = parseNodeMajor(version);
|
|
37
|
+
if (major === null || major < MIN_NODE_MAJOR) {
|
|
38
|
+
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for the model worker; ${candidate} reported ${version}`);
|
|
39
|
+
}
|
|
40
|
+
return candidate;
|
|
41
|
+
}
|
|
21
42
|
function getNodeExecPath() {
|
|
43
|
+
if (resolvedNodeExecPath)
|
|
44
|
+
return resolvedNodeExecPath;
|
|
22
45
|
const configured = process.env.PI_KNOWLEDGE_NODE_PATH?.trim();
|
|
23
|
-
if (configured)
|
|
24
|
-
|
|
46
|
+
if (configured) {
|
|
47
|
+
resolvedNodeExecPath = validateNodeExecPath(configured);
|
|
48
|
+
return resolvedNodeExecPath;
|
|
49
|
+
}
|
|
25
50
|
const execName = basename(process.execPath).toLowerCase();
|
|
26
|
-
|
|
51
|
+
const currentProcessLooksLikeNode = execName === "node" || execName === "node.exe";
|
|
52
|
+
if (currentProcessLooksLikeNode) {
|
|
53
|
+
const major = parseNodeMajor(process.version);
|
|
54
|
+
if (major !== null && major >= MIN_NODE_MAJOR) {
|
|
55
|
+
resolvedNodeExecPath = process.execPath;
|
|
56
|
+
return resolvedNodeExecPath;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const envNode = process.env.NODE?.trim();
|
|
60
|
+
for (const candidate of [envNode, "node"].filter((value) => Boolean(value))) {
|
|
61
|
+
try {
|
|
62
|
+
resolvedNodeExecPath = validateNodeExecPath(candidate);
|
|
63
|
+
return resolvedNodeExecPath;
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
// Try the next candidate before surfacing a deterministic error below.
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
throw new Error(`Node ${MIN_NODE_MAJOR}+ is required for local pi-knowledge embeddings. Set PI_KNOWLEDGE_NODE_PATH to a working node.exe path.`);
|
|
27
70
|
}
|
|
28
71
|
function appendWorkerStderr(current, chunk) {
|
|
29
72
|
const next = `${current}${chunk.toString("utf-8")}`;
|
|
@@ -36,92 +79,211 @@ function formatWorkerExitError(code, signal, stderrTail) {
|
|
|
36
79
|
return new Error(`${reason}. Set PI_KNOWLEDGE_NODE_PATH to a working Node binary if Pi is not running under Node.`);
|
|
37
80
|
return new Error(`${reason}. Worker stderr:\n${stderr}`);
|
|
38
81
|
}
|
|
39
|
-
function
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
let stderrTail = "";
|
|
51
|
-
child.stderr?.on("data", (chunk) => {
|
|
52
|
-
stderrTail = appendWorkerStderr(stderrTail, chunk);
|
|
53
|
-
});
|
|
54
|
-
worker.on("message", (message) => {
|
|
55
|
-
const request = pending.get(message.id);
|
|
56
|
-
if (!request)
|
|
57
|
-
return;
|
|
58
|
-
pending.delete(message.id);
|
|
59
|
-
if (message.error) {
|
|
60
|
-
request.reject(new Error(message.error));
|
|
61
|
-
}
|
|
62
|
-
else {
|
|
63
|
-
request.resolve(message.result);
|
|
64
|
-
}
|
|
65
|
-
});
|
|
66
|
-
worker.on("exit", (code, signal) => {
|
|
67
|
-
if (worker !== child)
|
|
68
|
-
return;
|
|
69
|
-
worker = null;
|
|
70
|
-
if (pending.size > 0) {
|
|
71
|
-
rejectPending(formatWorkerExitError(code, signal, stderrTail));
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
worker.on("error", (error) => {
|
|
75
|
-
if (worker !== child)
|
|
76
|
-
return;
|
|
77
|
-
worker = null;
|
|
78
|
-
rejectPending(error);
|
|
79
|
-
});
|
|
80
|
-
return worker;
|
|
82
|
+
function formatWorkerSpawnError(error, stderrTail) {
|
|
83
|
+
const stderr = stderrTail.trim();
|
|
84
|
+
const hint = "Set PI_KNOWLEDGE_NODE_PATH to a working Node 22+ binary if Pi or OMP is not running under Node.";
|
|
85
|
+
return new Error(`Model worker failed to start: ${error.message}. ${hint}${stderr ? ` Worker stderr:\n${stderr}` : ""}`);
|
|
86
|
+
}
|
|
87
|
+
function isWorkerResponse(message) {
|
|
88
|
+
if (typeof message !== "object" || message === null)
|
|
89
|
+
return false;
|
|
90
|
+
if (!("id" in message) || typeof message.id !== "number")
|
|
91
|
+
return false;
|
|
92
|
+
return !("error" in message) || message.error === undefined || typeof message.error === "string";
|
|
81
93
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
94
|
+
class ModelWorkerTransport {
|
|
95
|
+
child;
|
|
96
|
+
pending = new Map();
|
|
97
|
+
stderrTail = "";
|
|
98
|
+
exited = false;
|
|
99
|
+
spawnError = null;
|
|
100
|
+
constructor(child) {
|
|
101
|
+
this.child = child;
|
|
102
|
+
child.stderr?.on("data", (chunk) => {
|
|
103
|
+
this.stderrTail = appendWorkerStderr(this.stderrTail, chunk);
|
|
104
|
+
});
|
|
105
|
+
child.on("exit", (code, signal) => {
|
|
106
|
+
this.exited = true;
|
|
107
|
+
if (worker === this)
|
|
108
|
+
worker = null;
|
|
109
|
+
if (this.pending.size > 0)
|
|
110
|
+
this.rejectPending(formatWorkerExitError(code, signal, this.stderrTail));
|
|
111
|
+
});
|
|
112
|
+
child.on("error", (error) => {
|
|
113
|
+
this.spawnError = error;
|
|
114
|
+
this.exited = true;
|
|
115
|
+
if (worker === this)
|
|
116
|
+
worker = null;
|
|
117
|
+
this.rejectPending(formatWorkerSpawnError(error, this.stderrTail));
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
isConnected() {
|
|
121
|
+
return !this.exited && !this.child.killed && !this.spawnError && this.isTransportConnected();
|
|
122
|
+
}
|
|
123
|
+
request(message, signal) {
|
|
124
|
+
if (signal?.aborted)
|
|
125
|
+
throw new Error("Cancelled");
|
|
126
|
+
if (!this.isConnected())
|
|
127
|
+
throw new Error("Model worker is not connected");
|
|
128
|
+
const id = nextRequestId++;
|
|
129
|
+
const { promise, resolve, reject } = Promise.withResolvers();
|
|
88
130
|
let abortHandler;
|
|
89
131
|
const cleanup = () => {
|
|
90
132
|
if (abortHandler)
|
|
91
133
|
signal?.removeEventListener("abort", abortHandler);
|
|
92
134
|
};
|
|
93
|
-
pending.set(id, { resolve, reject });
|
|
94
135
|
if (signal) {
|
|
95
136
|
abortHandler = () => {
|
|
96
|
-
pending.delete(id);
|
|
137
|
+
this.pending.delete(id);
|
|
97
138
|
cleanup();
|
|
98
139
|
reject(new Error("Cancelled"));
|
|
99
140
|
};
|
|
100
141
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
101
142
|
}
|
|
102
|
-
|
|
103
|
-
const originalReject = reject;
|
|
104
|
-
pending.set(id, {
|
|
143
|
+
this.pending.set(id, {
|
|
105
144
|
resolve(value) {
|
|
106
145
|
cleanup();
|
|
107
|
-
|
|
146
|
+
resolve(value);
|
|
108
147
|
},
|
|
109
148
|
reject(error) {
|
|
110
149
|
cleanup();
|
|
111
|
-
|
|
150
|
+
reject(error);
|
|
112
151
|
},
|
|
113
152
|
});
|
|
114
|
-
|
|
153
|
+
this.writeRequest({ id, ...message }, (error) => {
|
|
115
154
|
if (!error)
|
|
116
155
|
return;
|
|
117
|
-
pending.delete(id);
|
|
156
|
+
this.pending.delete(id);
|
|
118
157
|
cleanup();
|
|
119
158
|
reject(error);
|
|
120
159
|
});
|
|
121
|
-
|
|
160
|
+
return promise;
|
|
161
|
+
}
|
|
162
|
+
shutdown() {
|
|
163
|
+
this.rejectPending(new Error("Model worker shut down"));
|
|
164
|
+
if (!this.child.killed)
|
|
165
|
+
this.child.kill("SIGKILL");
|
|
166
|
+
}
|
|
167
|
+
handleWorkerResponse(message) {
|
|
168
|
+
if (!isWorkerResponse(message)) {
|
|
169
|
+
this.rejectPending(new Error("Invalid model worker response"));
|
|
170
|
+
if (!this.child.killed)
|
|
171
|
+
this.child.kill("SIGKILL");
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const request = this.pending.get(message.id);
|
|
175
|
+
if (!request)
|
|
176
|
+
return;
|
|
177
|
+
this.pending.delete(message.id);
|
|
178
|
+
if (message.error) {
|
|
179
|
+
request.reject(new Error(message.error));
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
request.resolve(message.result);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
rejectPending(error) {
|
|
186
|
+
for (const request of this.pending.values()) {
|
|
187
|
+
request.reject(error);
|
|
188
|
+
}
|
|
189
|
+
this.pending.clear();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
class IpcModelWorkerTransport extends ModelWorkerTransport {
|
|
193
|
+
ipcChild;
|
|
194
|
+
constructor(ipcChild) {
|
|
195
|
+
super(ipcChild);
|
|
196
|
+
this.ipcChild = ipcChild;
|
|
197
|
+
ipcChild.on("message", (message) => this.handleWorkerResponse(message));
|
|
198
|
+
}
|
|
199
|
+
static create(workerPath) {
|
|
200
|
+
let child;
|
|
201
|
+
try {
|
|
202
|
+
child = fork(workerPath, {
|
|
203
|
+
execPath: getNodeExecPath(),
|
|
204
|
+
execArgv: getWorkerExecArgv(),
|
|
205
|
+
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
206
|
+
env: process.env,
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
return null;
|
|
211
|
+
}
|
|
212
|
+
if (typeof child.send !== "function") {
|
|
213
|
+
if (!child.killed)
|
|
214
|
+
child.kill("SIGKILL");
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
return new IpcModelWorkerTransport(child);
|
|
218
|
+
}
|
|
219
|
+
isTransportConnected() {
|
|
220
|
+
return this.ipcChild.connected;
|
|
221
|
+
}
|
|
222
|
+
writeRequest(message, callback) {
|
|
223
|
+
this.ipcChild.send(message, callback);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
class StdioModelWorkerTransport extends ModelWorkerTransport {
|
|
227
|
+
decoder = new StringDecoder("utf8");
|
|
228
|
+
stdoutBuffer = "";
|
|
229
|
+
constructor(workerPath) {
|
|
230
|
+
const child = spawn(getNodeExecPath(), [...getWorkerExecArgv(), workerPath, "--stdio"], {
|
|
231
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
232
|
+
env: process.env,
|
|
233
|
+
windowsHide: true,
|
|
234
|
+
});
|
|
235
|
+
super(child);
|
|
236
|
+
if (!child.stdin || !child.stdout) {
|
|
237
|
+
if (!child.killed)
|
|
238
|
+
child.kill("SIGKILL");
|
|
239
|
+
throw new Error("Model worker stdio pipes are unavailable");
|
|
240
|
+
}
|
|
241
|
+
child.stdout.on("data", (chunk) => this.handleStdoutChunk(chunk));
|
|
242
|
+
}
|
|
243
|
+
isTransportConnected() {
|
|
244
|
+
return this.child.stdin?.writable === true && this.child.stdin.destroyed !== true;
|
|
245
|
+
}
|
|
246
|
+
writeRequest(message, callback) {
|
|
247
|
+
const stdin = this.child.stdin;
|
|
248
|
+
if (!stdin || stdin.destroyed || !stdin.writable) {
|
|
249
|
+
callback(new Error("Model worker stdin is not writable"));
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
stdin.write(`${JSON.stringify(message)}\n`, "utf8", (error) => {
|
|
253
|
+
callback(error ?? null);
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
handleStdoutChunk(chunk) {
|
|
257
|
+
this.stdoutBuffer += this.decoder.write(chunk);
|
|
258
|
+
let newlineIndex = this.stdoutBuffer.indexOf("\n");
|
|
259
|
+
while (newlineIndex >= 0) {
|
|
260
|
+
const line = this.stdoutBuffer.slice(0, newlineIndex).trim();
|
|
261
|
+
this.stdoutBuffer = this.stdoutBuffer.slice(newlineIndex + 1);
|
|
262
|
+
if (line)
|
|
263
|
+
this.handleStdoutLine(line);
|
|
264
|
+
newlineIndex = this.stdoutBuffer.indexOf("\n");
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
handleStdoutLine(line) {
|
|
268
|
+
try {
|
|
269
|
+
this.handleWorkerResponse(JSON.parse(line));
|
|
270
|
+
}
|
|
271
|
+
catch (error) {
|
|
272
|
+
this.rejectPending(new Error(`Invalid model worker stdio response: ${error instanceof Error ? error.message : String(error)}`));
|
|
273
|
+
if (!this.child.killed)
|
|
274
|
+
this.child.kill("SIGKILL");
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
function getWorker() {
|
|
279
|
+
if (worker?.isConnected())
|
|
280
|
+
return worker;
|
|
281
|
+
const workerPath = getWorkerPath();
|
|
282
|
+
worker = IpcModelWorkerTransport.create(workerPath) ?? new StdioModelWorkerTransport(workerPath);
|
|
283
|
+
return worker;
|
|
122
284
|
}
|
|
123
285
|
export async function embedInModelWorker(texts, prefix, signal) {
|
|
124
|
-
const result = await
|
|
286
|
+
const result = await getWorker().request({ type: "embed", texts, prefix }, signal);
|
|
125
287
|
if (!Array.isArray(result))
|
|
126
288
|
throw new Error("Invalid embedding worker response");
|
|
127
289
|
return result.map((vector) => {
|
|
@@ -131,7 +293,7 @@ export async function embedInModelWorker(texts, prefix, signal) {
|
|
|
131
293
|
});
|
|
132
294
|
}
|
|
133
295
|
export async function rerankInModelWorker(query, candidates, topK, signal) {
|
|
134
|
-
const result = await
|
|
296
|
+
const result = await getWorker().request({ type: "rerank", query, candidates, topK }, signal);
|
|
135
297
|
if (!Array.isArray(result))
|
|
136
298
|
throw new Error("Invalid reranker worker response");
|
|
137
299
|
return result.map((item) => {
|
|
@@ -147,8 +309,5 @@ export async function rerankInModelWorker(query, candidates, topK, signal) {
|
|
|
147
309
|
export function shutdownModelWorker() {
|
|
148
310
|
const child = worker;
|
|
149
311
|
worker = null;
|
|
150
|
-
|
|
151
|
-
if (child && !child.killed) {
|
|
152
|
-
child.kill("SIGKILL");
|
|
153
|
-
}
|
|
312
|
+
child?.shutdown();
|
|
154
313
|
}
|
package/dist/src/model-worker.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { join } from "node:path";
|
|
2
|
+
import { createInterface } from "node:readline";
|
|
2
3
|
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
3
4
|
let embeddingPipeline = null;
|
|
4
5
|
let rerankerPipeline = null;
|
|
@@ -56,14 +57,98 @@ async function handleRerank(request) {
|
|
|
56
57
|
results.sort((a, b) => b.score - a.score);
|
|
57
58
|
return results.slice(0, request.topK);
|
|
58
59
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
function isStringArray(value) {
|
|
61
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
62
|
+
}
|
|
63
|
+
function isRerankCandidateArray(value) {
|
|
64
|
+
return (Array.isArray(value) &&
|
|
65
|
+
value.every((item) => typeof item === "object" &&
|
|
66
|
+
item !== null &&
|
|
67
|
+
"chunkId" in item &&
|
|
68
|
+
typeof item.chunkId === "string" &&
|
|
69
|
+
"content" in item &&
|
|
70
|
+
typeof item.content === "string"));
|
|
71
|
+
}
|
|
72
|
+
function parseWorkerRequest(message) {
|
|
73
|
+
if (typeof message !== "object" || message === null || !("id" in message) || typeof message.id !== "number") {
|
|
74
|
+
throw new Error("Invalid worker request id");
|
|
75
|
+
}
|
|
76
|
+
if (!("type" in message))
|
|
77
|
+
throw new Error("Invalid worker request type");
|
|
78
|
+
if (message.type === "embed") {
|
|
79
|
+
if (!("texts" in message) || !isStringArray(message.texts))
|
|
80
|
+
throw new Error("Invalid embed texts");
|
|
81
|
+
if (!("prefix" in message) || (message.prefix !== "query" && message.prefix !== "passage")) {
|
|
82
|
+
throw new Error("Invalid embed prefix");
|
|
83
|
+
}
|
|
84
|
+
return { id: message.id, type: "embed", texts: message.texts, prefix: message.prefix };
|
|
85
|
+
}
|
|
86
|
+
if (message.type === "rerank") {
|
|
87
|
+
if (!("query" in message) || typeof message.query !== "string")
|
|
88
|
+
throw new Error("Invalid rerank query");
|
|
89
|
+
if (!("topK" in message) || typeof message.topK !== "number")
|
|
90
|
+
throw new Error("Invalid rerank topK");
|
|
91
|
+
if (!("candidates" in message) || !isRerankCandidateArray(message.candidates)) {
|
|
92
|
+
throw new Error("Invalid rerank candidates");
|
|
64
93
|
}
|
|
65
|
-
|
|
66
|
-
|
|
94
|
+
return {
|
|
95
|
+
id: message.id,
|
|
96
|
+
type: "rerank",
|
|
97
|
+
query: message.query,
|
|
98
|
+
candidates: message.candidates,
|
|
99
|
+
topK: message.topK,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
throw new Error("Unsupported worker request type");
|
|
103
|
+
}
|
|
104
|
+
async function handleRequestMessage(message) {
|
|
105
|
+
let requestId = 0;
|
|
106
|
+
try {
|
|
107
|
+
if (typeof message === "object" && message !== null && "id" in message && typeof message.id === "number") {
|
|
108
|
+
requestId = message.id;
|
|
67
109
|
}
|
|
68
|
-
|
|
69
|
-
|
|
110
|
+
const request = parseWorkerRequest(message);
|
|
111
|
+
const result = request.type === "embed" ? await handleEmbed(request) : await handleRerank(request);
|
|
112
|
+
return { id: request.id, result };
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
return { id: requestId, error: error instanceof Error ? error.message : String(error) };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function writeStdioResponse(response) {
|
|
119
|
+
process.stdout.write(`${JSON.stringify(response)}\n`);
|
|
120
|
+
}
|
|
121
|
+
function runIpcMode() {
|
|
122
|
+
process.on("message", (message) => {
|
|
123
|
+
void (async () => {
|
|
124
|
+
const response = await handleRequestMessage(message);
|
|
125
|
+
process.send?.(response);
|
|
126
|
+
})();
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
function redirectConsoleOutputToStderr() {
|
|
130
|
+
const writeToStderr = (...args) => console.error(...args);
|
|
131
|
+
console.log = writeToStderr;
|
|
132
|
+
console.info = writeToStderr;
|
|
133
|
+
console.debug = writeToStderr;
|
|
134
|
+
}
|
|
135
|
+
function runStdioMode() {
|
|
136
|
+
redirectConsoleOutputToStderr();
|
|
137
|
+
const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
|
|
138
|
+
lines.on("line", (line) => {
|
|
139
|
+
void (async () => {
|
|
140
|
+
try {
|
|
141
|
+
writeStdioResponse(await handleRequestMessage(JSON.parse(line)));
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
writeStdioResponse({ id: 0, error: error instanceof Error ? error.message : String(error) });
|
|
145
|
+
}
|
|
146
|
+
})();
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
if (process.argv.includes("--stdio")) {
|
|
150
|
+
runStdioMode();
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
runIpcMode();
|
|
154
|
+
}
|
|
@@ -66,6 +66,7 @@ export declare function getDefaultKnowledgeDir(): string;
|
|
|
66
66
|
export declare function resolveHostKnowledgeDir(hostRoot: string, options?: {
|
|
67
67
|
legacyPiDir?: string;
|
|
68
68
|
exists?: (path: string) => boolean;
|
|
69
|
+
homeDir?: string;
|
|
69
70
|
}): string;
|
|
70
71
|
export declare function openDatabase(knowledgeDir?: string): Database.Database;
|
|
71
72
|
export declare function createKB(db: Database.Database, opts: {
|
|
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { existsSync, mkdirSync } from "node:fs";
|
|
3
3
|
import { createRequire } from "node:module";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
|
-
import {
|
|
5
|
+
import { dirname, join } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
const SCHEMA_VERSION = 4;
|
|
8
8
|
const ITERATION_BATCH_SIZE = 500;
|
|
@@ -174,6 +174,20 @@ function applyPragma(db, pragma) {
|
|
|
174
174
|
}
|
|
175
175
|
db.exec(`PRAGMA ${pragma}`);
|
|
176
176
|
}
|
|
177
|
+
function basenameForHostPath(path) {
|
|
178
|
+
const normalized = path.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
179
|
+
const slashIndex = normalized.lastIndexOf("/");
|
|
180
|
+
return slashIndex >= 0 ? normalized.slice(slashIndex + 1) : normalized;
|
|
181
|
+
}
|
|
182
|
+
function normalizeForHostComparison(path) {
|
|
183
|
+
const normalized = path.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
184
|
+
return /^[A-Za-z]:\//.test(normalized) ? normalized.toLowerCase() : normalized;
|
|
185
|
+
}
|
|
186
|
+
function isWithinOrEqualPath(path, parent) {
|
|
187
|
+
const normalizedPath = normalizeForHostComparison(path);
|
|
188
|
+
const normalizedParent = normalizeForHostComparison(parent);
|
|
189
|
+
return normalizedPath === normalizedParent || normalizedPath.startsWith(`${normalizedParent}/`);
|
|
190
|
+
}
|
|
177
191
|
export function getDefaultKnowledgeDir() {
|
|
178
192
|
const explicit = process.env.PI_KNOWLEDGE_DIR?.trim() || process.env.OMP_KNOWLEDGE_DIR?.trim();
|
|
179
193
|
if (explicit)
|
|
@@ -187,9 +201,10 @@ export function getDefaultKnowledgeDir() {
|
|
|
187
201
|
export function resolveHostKnowledgeDir(hostRoot, options = {}) {
|
|
188
202
|
const pathExists = options.exists ?? existsSync;
|
|
189
203
|
const target = join(hostRoot, "knowledge");
|
|
190
|
-
const
|
|
191
|
-
|
|
192
|
-
|
|
204
|
+
const homeDir = options.homeDir ?? homedir();
|
|
205
|
+
const legacyPiDir = options.legacyPiDir ?? join(homeDir, ".pi", "knowledge");
|
|
206
|
+
if (basenameForHostPath(hostRoot) === ".omp" &&
|
|
207
|
+
isWithinOrEqualPath(hostRoot, homeDir) &&
|
|
193
208
|
!pathExists(target) &&
|
|
194
209
|
pathExists(legacyPiDir)) {
|
|
195
210
|
return legacyPiDir;
|
|
@@ -200,7 +215,10 @@ function isOmpHost() {
|
|
|
200
215
|
if (process.env.OMP_PROFILE?.trim())
|
|
201
216
|
return true;
|
|
202
217
|
const candidates = [process.argv[1], process.execPath].filter((value) => typeof value === "string");
|
|
203
|
-
return candidates.some((value) =>
|
|
218
|
+
return candidates.some((value) => {
|
|
219
|
+
const executableName = basenameForHostPath(value).toLowerCase();
|
|
220
|
+
return executableName === "omp" || executableName === "omp.exe";
|
|
221
|
+
});
|
|
204
222
|
}
|
|
205
223
|
export function openDatabase(knowledgeDir) {
|
|
206
224
|
const dir = knowledgeDir ?? getDefaultKnowledgeDir();
|
package/docs/configuration.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
| `OMP_CODING_AGENT_DIR` | unset | Derive the OMP host root from a configured agent directory. Used when `PI_CODING_AGENT_DIR` is not set. |
|
|
13
13
|
| `OMP_PROFILE` | unset | Treat the current process as OMP-hosted for default path selection. |
|
|
14
14
|
| `PI_KNOWLEDGE_MODEL_CACHE_DIR` | `<knowledge-dir>/models` | Override the local Transformers.js model cache directory. |
|
|
15
|
-
| `PI_KNOWLEDGE_NODE_PATH` | current Node when possible,
|
|
15
|
+
| `PI_KNOWLEDGE_NODE_PATH` | current Node when possible, then `NODE`, then `node` on `PATH` | Override the Node 22+ executable used to run the isolated model worker. Useful for Windows OMP packaged runtimes where the host process is not Node. |
|
|
16
16
|
|
|
17
17
|
Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under OMP. For the default home OMP root, `pi-knowledge` preserves an existing legacy `~/.pi/knowledge` directory when `~/.omp/knowledge` does not exist, so existing Pi knowledge bases remain visible during migration.
|
|
18
18
|
|
|
@@ -30,6 +30,8 @@ Default data storage is `~/.pi/knowledge` under Pi and `~/.omp/knowledge` under
|
|
|
30
30
|
| `PI_KNOWLEDGE_EMBEDDING_IDLE_MS` | `30000` | Idle timer used only when native idle disposal coordination is enabled. Mainly for lifecycle stress tests. |
|
|
31
31
|
| `PI_KNOWLEDGE_OFFLINE` | unset | Use with a pre-populated model cache for offline local model operation. See `docs/offline-mode.md`. |
|
|
32
32
|
|
|
33
|
+
Local embeddings require a real Node 22+ executable for the isolated model worker. `pi-knowledge` first tries Node IPC and automatically falls back to a stdin/stdout JSONL worker transport when the host runtime does not expose `child_process.fork().send()`, which can happen in Windows OMP compatibility layers. If worker startup fails, set `PI_KNOWLEDGE_NODE_PATH` to the full `node.exe` path or use an OpenAI-compatible embedding provider.
|
|
34
|
+
|
|
33
35
|
API embedding failures intentionally surface by default. Silent fallback can hide bad API keys, wrong base URLs, unsupported model names, or context-window errors and can produce a KB with a different embedding model than intended.
|
|
34
36
|
|
|
35
37
|
## Runtime Features and Diagnostics
|
|
@@ -65,7 +67,7 @@ Compatibility guarantees:
|
|
|
65
67
|
|
|
66
68
|
- The packaged `extension.js` loads built `dist/index.js` when present and falls back to source `index.ts` for local development.
|
|
67
69
|
- Runtime modules are imported lazily, after the extension is actually used.
|
|
68
|
-
- Local embedding and reranking models run in an isolated model worker, not in the Pi or OMP TUI process.
|
|
70
|
+
- Local embedding and reranking models run in an isolated Node model worker, not in the Pi or OMP TUI process. Node IPC is preferred; stdin/stdout JSONL transport is used automatically when host IPC is unavailable.
|
|
69
71
|
- Native SQLite loading includes a fallback for hoisted plugin dependency layouts.
|
|
70
72
|
- OMP path resolution can use `OMP_KNOWLEDGE_DIR`, `OMP_CODING_AGENT_DIR`, and `OMP_PROFILE`.
|
|
71
73
|
- Existing default Pi knowledge data can remain visible under the default OMP home root through the legacy `~/.pi/knowledge` fallback described above.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-knowledge",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.3",
|
|
4
4
|
"description": "Local-first RAG knowledge base for Pi and OMP coding agents with hybrid code/doc search, reranking, diagnostics, and large-project indexing",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -75,6 +75,6 @@
|
|
|
75
75
|
"license": "MIT",
|
|
76
76
|
"repository": {
|
|
77
77
|
"type": "git",
|
|
78
|
-
"url": "https://github.com/nczz/pi-knowledge.git"
|
|
78
|
+
"url": "git+https://github.com/nczz/pi-knowledge.git"
|
|
79
79
|
}
|
|
80
80
|
}
|