pi-knowledge 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +63 -24
- package/dist/index.d.ts +10 -4
- package/dist/index.js +300 -187
- package/dist/src/coverage-snapshot.d.ts +23 -0
- package/dist/src/coverage-snapshot.js +53 -0
- package/dist/src/daemon/client.d.ts +60 -0
- package/dist/src/daemon/client.js +360 -0
- package/dist/src/daemon/main.d.ts +2 -0
- package/dist/src/daemon/main.js +109 -0
- package/dist/src/daemon/protocol.d.ts +71 -0
- package/dist/src/daemon/protocol.js +170 -0
- package/dist/src/daemon/runtime.d.ts +26 -0
- package/dist/src/daemon/runtime.js +192 -0
- package/dist/src/daemon/server.d.ts +46 -0
- package/dist/src/daemon/server.js +500 -0
- package/dist/src/daemon/service.d.ts +49 -0
- package/dist/src/daemon/service.js +527 -0
- package/dist/src/diagnostics/health.d.ts +6 -0
- package/dist/src/diagnostics/health.js +135 -2
- package/dist/src/engine.d.ts +63 -9
- package/dist/src/engine.js +959 -170
- package/dist/src/filesystem-access.d.ts +9 -0
- package/dist/src/filesystem-access.js +100 -0
- package/dist/src/indexer/chunker.d.ts +10 -0
- package/dist/src/indexer/chunker.js +284 -9
- package/dist/src/mcp/http.d.ts +47 -0
- package/dist/src/mcp/http.js +564 -0
- package/dist/src/mcp/main.d.ts +2 -0
- package/dist/src/mcp/main.js +97 -0
- package/dist/src/mcp/protocol.d.ts +81 -0
- package/dist/src/mcp/protocol.js +444 -0
- package/dist/src/mcp/server.d.ts +42 -0
- package/dist/src/mcp/server.js +378 -0
- package/dist/src/model-worker-client.js +9 -0
- package/dist/src/model-worker-contract.d.ts +3 -0
- package/dist/src/model-worker-contract.js +8 -0
- package/dist/src/model-worker.js +76 -4
- package/dist/src/network/safe-fetch.d.ts +7 -0
- package/dist/src/network/safe-fetch.js +199 -0
- package/dist/src/node-runtime.d.ts +3 -0
- package/dist/src/node-runtime.js +78 -0
- package/dist/src/profiles.d.ts +15 -0
- package/dist/src/profiles.js +160 -0
- package/dist/src/runtime-config.js +5 -6
- package/dist/src/search/tuning.d.ts +2 -0
- package/dist/src/search/tuning.js +2 -0
- package/dist/src/secure-files.d.ts +17 -0
- package/dist/src/secure-files.js +184 -0
- package/dist/src/status.js +12 -7
- package/dist/src/storage/sqlite.d.ts +4 -1
- package/dist/src/storage/sqlite.js +46 -3
- package/dist/src/type-guards.d.ts +1 -0
- package/dist/src/type-guards.js +3 -0
- package/dist/src/watcher/file-watcher.d.ts +3 -1
- package/dist/src/watcher/file-watcher.js +166 -32
- package/dist/src/windows-security.d.ts +4 -0
- package/dist/src/windows-security.js +47 -0
- package/docs/configuration.md +42 -10
- package/docs/daemon-profiles.md +302 -0
- package/docs/mcp-clients.md +266 -0
- package/docs/offline-mode.md +6 -6
- package/package.json +28 -2
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
import { existsSync, lstatSync, realpathSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { assertOwnedDirectory } from "../secure-files.js";
|
|
4
|
+
import { isObjectRecord } from "../type-guards.js";
|
|
5
|
+
import { MAX_MCP_CONCURRENT_REQUESTS, MAX_MCP_INPUT_BYTES, MAX_MCP_OUTPUT_BYTES, MAX_MCP_OUTPUT_QUEUE_BYTES, MCP_SHUTDOWN_TIMEOUT_MS, McpProtocolError, McpProtocolSession, mcpErrorMessage, mcpRpcError, parseMcpRequest, } from "./protocol.js";
|
|
6
|
+
async function settlesWithin(promise, timeoutMs) {
|
|
7
|
+
let timer;
|
|
8
|
+
const timeout = new Promise((resolveTimeout) => {
|
|
9
|
+
timer = setTimeout(() => resolveTimeout(false), timeoutMs);
|
|
10
|
+
timer.unref();
|
|
11
|
+
});
|
|
12
|
+
const settled = promise.then(() => true, () => true);
|
|
13
|
+
try {
|
|
14
|
+
return await Promise.race([settled, timeout]);
|
|
15
|
+
}
|
|
16
|
+
finally {
|
|
17
|
+
if (timer)
|
|
18
|
+
clearTimeout(timer);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function parsePort(value) {
|
|
22
|
+
if (!/^\d+$/.test(value))
|
|
23
|
+
throw new Error("--port must be an integer from 0 to 65535");
|
|
24
|
+
const port = Number(value);
|
|
25
|
+
if (!Number.isSafeInteger(port) || port < 0 || port > 65_535) {
|
|
26
|
+
throw new Error("--port must be an integer from 0 to 65535");
|
|
27
|
+
}
|
|
28
|
+
return port;
|
|
29
|
+
}
|
|
30
|
+
function normalizeEndpoint(value) {
|
|
31
|
+
if (!value.startsWith("/") || value.startsWith("//") || /[?#\s]/.test(value)) {
|
|
32
|
+
throw new Error("--endpoint must be one absolute HTTP path without query or fragment");
|
|
33
|
+
}
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
function normalizeHost(value) {
|
|
37
|
+
if (value !== "127.0.0.1" && value !== "::1" && value !== "localhost") {
|
|
38
|
+
throw new Error("--host must be 127.0.0.1, ::1, or localhost");
|
|
39
|
+
}
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
42
|
+
function normalizeOrigin(value) {
|
|
43
|
+
let parsed;
|
|
44
|
+
try {
|
|
45
|
+
parsed = new URL(value);
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
throw new Error(`Invalid --allow-origin value: ${value}`);
|
|
49
|
+
}
|
|
50
|
+
if ((parsed.protocol !== "http:" && parsed.protocol !== "https:") ||
|
|
51
|
+
parsed.username ||
|
|
52
|
+
parsed.password ||
|
|
53
|
+
parsed.pathname !== "/" ||
|
|
54
|
+
parsed.search ||
|
|
55
|
+
parsed.hash) {
|
|
56
|
+
throw new Error(`--allow-origin must be an HTTP origin without path, credentials, query, or fragment: ${value}`);
|
|
57
|
+
}
|
|
58
|
+
return parsed.origin;
|
|
59
|
+
}
|
|
60
|
+
export function parseMcpArguments(argv) {
|
|
61
|
+
const options = {
|
|
62
|
+
allowRoots: [],
|
|
63
|
+
allowNetwork: false,
|
|
64
|
+
allowWrite: false,
|
|
65
|
+
allowAdmin: false,
|
|
66
|
+
transport: "stdio",
|
|
67
|
+
httpHost: "127.0.0.1",
|
|
68
|
+
httpPort: 3210,
|
|
69
|
+
httpEndpoint: "/mcp",
|
|
70
|
+
allowOrigins: [],
|
|
71
|
+
};
|
|
72
|
+
let transportSpecified = false;
|
|
73
|
+
let httpOptionSpecified = false;
|
|
74
|
+
for (let index = 0; index < argv.length; index++) {
|
|
75
|
+
const argument = argv[index];
|
|
76
|
+
if (argument === "--allow-network")
|
|
77
|
+
options.allowNetwork = true;
|
|
78
|
+
else if (argument === "--allow-write")
|
|
79
|
+
options.allowWrite = true;
|
|
80
|
+
else if (argument === "--allow-admin")
|
|
81
|
+
options.allowAdmin = true;
|
|
82
|
+
else if (argument === "--profile" ||
|
|
83
|
+
argument === "--allow-root" ||
|
|
84
|
+
argument === "--transport" ||
|
|
85
|
+
argument === "--host" ||
|
|
86
|
+
argument === "--port" ||
|
|
87
|
+
argument === "--endpoint" ||
|
|
88
|
+
argument === "--allow-origin") {
|
|
89
|
+
const value = argv[index + 1];
|
|
90
|
+
if (!value)
|
|
91
|
+
throw new Error(`Missing value for ${argument}`);
|
|
92
|
+
if (argument === "--profile") {
|
|
93
|
+
if (options.profileName !== undefined)
|
|
94
|
+
throw new Error("--profile can only be specified once");
|
|
95
|
+
options.profileName = value;
|
|
96
|
+
}
|
|
97
|
+
else if (argument === "--allow-root")
|
|
98
|
+
options.allowRoots.push(value);
|
|
99
|
+
else if (argument === "--transport") {
|
|
100
|
+
if (transportSpecified)
|
|
101
|
+
throw new Error("--transport can only be specified once");
|
|
102
|
+
if (value !== "stdio" && value !== "http")
|
|
103
|
+
throw new Error("--transport must be stdio or http");
|
|
104
|
+
options.transport = value;
|
|
105
|
+
transportSpecified = true;
|
|
106
|
+
}
|
|
107
|
+
else if (argument === "--host") {
|
|
108
|
+
options.httpHost = normalizeHost(value);
|
|
109
|
+
httpOptionSpecified = true;
|
|
110
|
+
}
|
|
111
|
+
else if (argument === "--port") {
|
|
112
|
+
options.httpPort = parsePort(value);
|
|
113
|
+
httpOptionSpecified = true;
|
|
114
|
+
}
|
|
115
|
+
else if (argument === "--endpoint") {
|
|
116
|
+
options.httpEndpoint = normalizeEndpoint(value);
|
|
117
|
+
httpOptionSpecified = true;
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
options.allowOrigins.push(normalizeOrigin(value));
|
|
121
|
+
httpOptionSpecified = true;
|
|
122
|
+
}
|
|
123
|
+
index++;
|
|
124
|
+
}
|
|
125
|
+
else
|
|
126
|
+
throw new Error(`Unknown MCP option: ${argument}`);
|
|
127
|
+
}
|
|
128
|
+
if (options.transport === "stdio" && httpOptionSpecified) {
|
|
129
|
+
throw new Error("--host, --port, --endpoint, and --allow-origin require --transport http");
|
|
130
|
+
}
|
|
131
|
+
options.allowOrigins = [...new Set(options.allowOrigins)].sort();
|
|
132
|
+
return options;
|
|
133
|
+
}
|
|
134
|
+
export function canonicalizeAllowedRoots(paths) {
|
|
135
|
+
const roots = new Set();
|
|
136
|
+
for (const input of paths) {
|
|
137
|
+
const absolute = resolve(input);
|
|
138
|
+
if (!existsSync(absolute))
|
|
139
|
+
throw new Error(`Allowed root does not exist: ${absolute}`);
|
|
140
|
+
if (lstatSync(absolute).isSymbolicLink()) {
|
|
141
|
+
throw new Error(`Allowed root must be a canonical directory: ${absolute}`);
|
|
142
|
+
}
|
|
143
|
+
const canonical = realpathSync.native(absolute);
|
|
144
|
+
assertOwnedDirectory(canonical);
|
|
145
|
+
roots.add(canonical);
|
|
146
|
+
}
|
|
147
|
+
return [...roots].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
|
148
|
+
}
|
|
149
|
+
class OutputSink {
|
|
150
|
+
chain = Promise.resolve();
|
|
151
|
+
queuedBytes = 0;
|
|
152
|
+
fatalError;
|
|
153
|
+
output;
|
|
154
|
+
onFatal;
|
|
155
|
+
constructor(output, onFatal) {
|
|
156
|
+
this.output = output;
|
|
157
|
+
this.onFatal = onFatal;
|
|
158
|
+
output.on("error", (error) => this.fail(error));
|
|
159
|
+
output.on("close", () => this.fail(new Error("MCP output closed")));
|
|
160
|
+
}
|
|
161
|
+
fail(error) {
|
|
162
|
+
if (this.fatalError)
|
|
163
|
+
return;
|
|
164
|
+
this.fatalError = error;
|
|
165
|
+
this.onFatal(error);
|
|
166
|
+
}
|
|
167
|
+
writeLine(line) {
|
|
168
|
+
if (this.fatalError)
|
|
169
|
+
return Promise.reject(this.fatalError);
|
|
170
|
+
if (this.output.destroyed || this.output.writableEnded)
|
|
171
|
+
return Promise.reject(new Error("MCP output is closed"));
|
|
172
|
+
const { promise, resolve: resolveWrite, reject } = Promise.withResolvers();
|
|
173
|
+
let settled = false;
|
|
174
|
+
const finish = (error) => {
|
|
175
|
+
if (settled)
|
|
176
|
+
return;
|
|
177
|
+
settled = true;
|
|
178
|
+
this.output.off("drain", onDrain);
|
|
179
|
+
this.output.off("error", onError);
|
|
180
|
+
this.output.off("close", onClose);
|
|
181
|
+
if (error)
|
|
182
|
+
reject(error);
|
|
183
|
+
else
|
|
184
|
+
resolveWrite();
|
|
185
|
+
};
|
|
186
|
+
const onDrain = () => finish();
|
|
187
|
+
const onError = (error) => finish(error);
|
|
188
|
+
const onClose = () => finish(new Error("MCP output closed before draining"));
|
|
189
|
+
this.output.once("error", onError);
|
|
190
|
+
this.output.once("close", onClose);
|
|
191
|
+
if (this.output.write(line))
|
|
192
|
+
finish();
|
|
193
|
+
else
|
|
194
|
+
this.output.once("drain", onDrain);
|
|
195
|
+
return promise;
|
|
196
|
+
}
|
|
197
|
+
send(value) {
|
|
198
|
+
const line = `${JSON.stringify(value)}\n`;
|
|
199
|
+
const bytes = Buffer.byteLength(line);
|
|
200
|
+
if (bytes > MAX_MCP_OUTPUT_BYTES)
|
|
201
|
+
return Promise.reject(new Error("MCP response exceeds the output frame limit"));
|
|
202
|
+
if (this.fatalError)
|
|
203
|
+
return Promise.reject(this.fatalError);
|
|
204
|
+
if (this.queuedBytes + bytes > MAX_MCP_OUTPUT_QUEUE_BYTES) {
|
|
205
|
+
const error = new Error("MCP output queue is full");
|
|
206
|
+
this.fail(error);
|
|
207
|
+
this.output.destroy(error);
|
|
208
|
+
return Promise.reject(error);
|
|
209
|
+
}
|
|
210
|
+
this.queuedBytes += bytes;
|
|
211
|
+
const write = this.chain.then(() => this.writeLine(line));
|
|
212
|
+
this.chain = write
|
|
213
|
+
.catch((error) => this.fail(error instanceof Error ? error : new Error(String(error))))
|
|
214
|
+
.finally(() => {
|
|
215
|
+
this.queuedBytes -= bytes;
|
|
216
|
+
});
|
|
217
|
+
return write;
|
|
218
|
+
}
|
|
219
|
+
abort(error) {
|
|
220
|
+
this.fail(error);
|
|
221
|
+
if (!this.output.destroyed)
|
|
222
|
+
this.output.destroy(error);
|
|
223
|
+
}
|
|
224
|
+
async flush() {
|
|
225
|
+
await this.chain;
|
|
226
|
+
if (this.fatalError)
|
|
227
|
+
throw this.fatalError;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
export class McpStdioAdapter {
|
|
231
|
+
output;
|
|
232
|
+
session;
|
|
233
|
+
active = new Set();
|
|
234
|
+
inputFrame = Buffer.allocUnsafe(MAX_MCP_INPUT_BYTES);
|
|
235
|
+
inputBytes = 0;
|
|
236
|
+
closing = false;
|
|
237
|
+
options;
|
|
238
|
+
overloadResponse;
|
|
239
|
+
constructor(options) {
|
|
240
|
+
this.options = options;
|
|
241
|
+
this.output = new OutputSink(options.output, (error) => {
|
|
242
|
+
this.closing = true;
|
|
243
|
+
options.input.destroy(error);
|
|
244
|
+
});
|
|
245
|
+
this.session = new McpProtocolSession({
|
|
246
|
+
...options,
|
|
247
|
+
onError: (error) => this.logError(error),
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
async sendError(id, code, message) {
|
|
251
|
+
await this.output.send(mcpRpcError(id, code, message));
|
|
252
|
+
}
|
|
253
|
+
logError(error) {
|
|
254
|
+
this.options.error.write(`${mcpErrorMessage(error)}\n`);
|
|
255
|
+
}
|
|
256
|
+
async handleValue(value) {
|
|
257
|
+
let request;
|
|
258
|
+
try {
|
|
259
|
+
request = parseMcpRequest(value);
|
|
260
|
+
}
|
|
261
|
+
catch (error) {
|
|
262
|
+
const code = error instanceof McpProtocolError ? error.code : -32600;
|
|
263
|
+
await this.sendError(null, code, mcpErrorMessage(error));
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
const handled = await this.session.handleRequest(request, (message) => this.output.send(message));
|
|
267
|
+
if (handled.response)
|
|
268
|
+
await this.output.send(handled.response);
|
|
269
|
+
}
|
|
270
|
+
rejectOverload(id, code, message) {
|
|
271
|
+
if (this.overloadResponse) {
|
|
272
|
+
this.closing = true;
|
|
273
|
+
this.options.input.destroy();
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const operation = this.sendError(id, code, message);
|
|
277
|
+
this.overloadResponse = operation;
|
|
278
|
+
this.active.add(operation);
|
|
279
|
+
void operation.then(() => {
|
|
280
|
+
this.active.delete(operation);
|
|
281
|
+
if (this.overloadResponse === operation)
|
|
282
|
+
this.overloadResponse = undefined;
|
|
283
|
+
}, (error) => {
|
|
284
|
+
this.active.delete(operation);
|
|
285
|
+
if (this.overloadResponse === operation)
|
|
286
|
+
this.overloadResponse = undefined;
|
|
287
|
+
this.logError(error);
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
acceptLine(line) {
|
|
291
|
+
let value;
|
|
292
|
+
try {
|
|
293
|
+
value = JSON.parse(line.toString("utf8"));
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
if (this.active.size >= MAX_MCP_CONCURRENT_REQUESTS) {
|
|
297
|
+
this.rejectOverload(null, -32700, mcpErrorMessage(error));
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const operation = this.sendError(null, -32700, mcpErrorMessage(error));
|
|
301
|
+
this.active.add(operation);
|
|
302
|
+
void operation.then(() => this.active.delete(operation), (candidate) => {
|
|
303
|
+
this.active.delete(operation);
|
|
304
|
+
this.logError(candidate);
|
|
305
|
+
});
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (isObjectRecord(value) &&
|
|
309
|
+
(value.method === "notifications/cancelled" || value.method === "notifications/initialized")) {
|
|
310
|
+
void this.handleValue(value).catch((error) => this.logError(error));
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
if (this.active.size >= MAX_MCP_CONCURRENT_REQUESTS) {
|
|
314
|
+
const id = isObjectRecord(value) && "id" in value ? value.id : null;
|
|
315
|
+
const responseId = typeof id === "string" || typeof id === "number" ? id : null;
|
|
316
|
+
this.rejectOverload(responseId, -32000, "MCP adapter request limit reached");
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const operation = this.handleValue(value);
|
|
320
|
+
this.active.add(operation);
|
|
321
|
+
void operation.catch((error) => this.logError(error)).finally(() => this.active.delete(operation));
|
|
322
|
+
}
|
|
323
|
+
consumeChunk(chunk) {
|
|
324
|
+
let offset = 0;
|
|
325
|
+
while (offset < chunk.byteLength) {
|
|
326
|
+
const newline = chunk.indexOf(0x0a, offset);
|
|
327
|
+
const end = newline < 0 ? chunk.byteLength : newline;
|
|
328
|
+
const part = chunk.subarray(offset, end);
|
|
329
|
+
if (this.inputBytes + part.byteLength > MAX_MCP_INPUT_BYTES) {
|
|
330
|
+
throw new Error("MCP input frame exceeds the configured limit");
|
|
331
|
+
}
|
|
332
|
+
part.copy(this.inputFrame, this.inputBytes);
|
|
333
|
+
this.inputBytes += part.byteLength;
|
|
334
|
+
if (newline < 0)
|
|
335
|
+
return;
|
|
336
|
+
const line = this.inputFrame.subarray(0, this.inputBytes);
|
|
337
|
+
this.inputBytes = 0;
|
|
338
|
+
if (line.toString("utf8").trim())
|
|
339
|
+
this.acceptLine(line);
|
|
340
|
+
offset = newline + 1;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
async run() {
|
|
344
|
+
let terminalError;
|
|
345
|
+
try {
|
|
346
|
+
for await (const chunk of this.options.input) {
|
|
347
|
+
if (this.closing)
|
|
348
|
+
break;
|
|
349
|
+
this.consumeChunk(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
350
|
+
}
|
|
351
|
+
const remainder = this.inputFrame.subarray(0, this.inputBytes);
|
|
352
|
+
if (remainder.toString("utf8").trim())
|
|
353
|
+
this.acceptLine(remainder);
|
|
354
|
+
}
|
|
355
|
+
catch (error) {
|
|
356
|
+
terminalError = error;
|
|
357
|
+
}
|
|
358
|
+
this.closing = true;
|
|
359
|
+
this.session.abortAll();
|
|
360
|
+
const cleanup = Promise.allSettled([...this.active]).then(() => this.output.flush());
|
|
361
|
+
if (!(await settlesWithin(cleanup, MCP_SHUTDOWN_TIMEOUT_MS))) {
|
|
362
|
+
const error = new Error("MCP shutdown timed out while draining output");
|
|
363
|
+
this.output.abort(error);
|
|
364
|
+
void cleanup.catch(() => undefined);
|
|
365
|
+
terminalError ??= error;
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
try {
|
|
369
|
+
await cleanup;
|
|
370
|
+
}
|
|
371
|
+
catch (error) {
|
|
372
|
+
terminalError ??= error;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
if (terminalError !== undefined)
|
|
376
|
+
throw terminalError;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
@@ -3,6 +3,7 @@ import { existsSync, readdirSync, statSync } from "node:fs";
|
|
|
3
3
|
import { basename, delimiter, join } from "node:path";
|
|
4
4
|
import { StringDecoder } from "node:string_decoder";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { assertExtractedDocumentSize, MAX_WORKER_STDIO_FRAME_BYTES } from "./model-worker-contract.js";
|
|
6
7
|
import { getConfiguredNodePath, writeRuntimeConfig } from "./runtime-config.js";
|
|
7
8
|
let worker = null;
|
|
8
9
|
let nextRequestId = 1;
|
|
@@ -229,6 +230,7 @@ class ModelWorkerTransport {
|
|
|
229
230
|
};
|
|
230
231
|
if (signal) {
|
|
231
232
|
abortHandler = () => {
|
|
233
|
+
this.writeRequest({ id, type: "cancel" }, () => { });
|
|
232
234
|
this.pending.delete(id);
|
|
233
235
|
cleanup();
|
|
234
236
|
reject(new Error("Cancelled"));
|
|
@@ -355,6 +357,12 @@ class StdioModelWorkerTransport extends ModelWorkerTransport {
|
|
|
355
357
|
});
|
|
356
358
|
}
|
|
357
359
|
handleStdoutChunk(chunk) {
|
|
360
|
+
if (Buffer.byteLength(this.stdoutBuffer) + chunk.byteLength > MAX_WORKER_STDIO_FRAME_BYTES) {
|
|
361
|
+
this.rejectPending(new Error("Model worker stdio response exceeds the configured frame limit"));
|
|
362
|
+
if (!this.child.killed)
|
|
363
|
+
this.child.kill("SIGKILL");
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
358
366
|
this.stdoutBuffer += this.decoder.write(chunk);
|
|
359
367
|
let newlineIndex = this.stdoutBuffer.indexOf("\n");
|
|
360
368
|
while (newlineIndex >= 0) {
|
|
@@ -423,6 +431,7 @@ export async function extractDocumentInModelWorker(filePath, fileType, signal) {
|
|
|
423
431
|
result.fileType !== fileType) {
|
|
424
432
|
throw new Error("Invalid document extraction worker response");
|
|
425
433
|
}
|
|
434
|
+
assertExtractedDocumentSize(result.content);
|
|
426
435
|
return { content: result.content, fileType };
|
|
427
436
|
}
|
|
428
437
|
export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const MAX_EXTRACTED_DOCUMENT_BYTES = 8 * 1024 * 1024;
|
|
2
|
+
export const MAX_WORKER_STDIO_FRAME_BYTES = 64 * 1024 * 1024;
|
|
3
|
+
export function assertExtractedDocumentSize(content) {
|
|
4
|
+
const bytes = Buffer.byteLength(content);
|
|
5
|
+
if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
|
|
6
|
+
throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
|
|
7
|
+
}
|
|
8
|
+
}
|
package/dist/src/model-worker.js
CHANGED
|
@@ -2,12 +2,19 @@ import { readFile } from "node:fs/promises";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { createInterface } from "node:readline";
|
|
4
4
|
import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
|
|
5
|
+
import { assertExtractedDocumentSize, MAX_EXTRACTED_DOCUMENT_BYTES, MAX_WORKER_STDIO_FRAME_BYTES, } from "./model-worker-contract.js";
|
|
5
6
|
import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
|
|
6
7
|
import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
|
|
7
8
|
import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
|
|
8
9
|
let embeddingPipeline = null;
|
|
9
10
|
let embeddingPipelinePromise = null;
|
|
10
11
|
let rerankerPipeline = null;
|
|
12
|
+
const cancelledRequestIds = new Set();
|
|
13
|
+
const activeRequestIds = new Set();
|
|
14
|
+
function throwIfCancelled(id) {
|
|
15
|
+
if (cancelledRequestIds.has(id))
|
|
16
|
+
throw new Error("Cancelled");
|
|
17
|
+
}
|
|
11
18
|
function getModelCacheDir() {
|
|
12
19
|
return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
|
|
13
20
|
}
|
|
@@ -82,13 +89,16 @@ function applyEmbeddingPrefix(text, role, prefixes) {
|
|
|
82
89
|
return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
|
|
83
90
|
}
|
|
84
91
|
async function handleEmbed(request) {
|
|
92
|
+
throwIfCancelled(request.id);
|
|
85
93
|
const pipe = await loadEmbeddingPipeline();
|
|
86
94
|
const vectors = [];
|
|
87
95
|
for (const text of request.texts) {
|
|
96
|
+
throwIfCancelled(request.id);
|
|
88
97
|
const output = await pipe(applyEmbeddingPrefix(text, request.prefix, request.prefixes), {
|
|
89
98
|
pooling: "mean",
|
|
90
99
|
normalize: true,
|
|
91
100
|
});
|
|
101
|
+
throwIfCancelled(request.id);
|
|
92
102
|
vectors.push(Array.from(output.data));
|
|
93
103
|
}
|
|
94
104
|
return vectors;
|
|
@@ -100,6 +110,7 @@ function embeddingMaximumTokens(tokenizer) {
|
|
|
100
110
|
: 512;
|
|
101
111
|
}
|
|
102
112
|
async function handleMeasureEmbeddingInput(request) {
|
|
113
|
+
throwIfCancelled(request.id);
|
|
103
114
|
const pipe = await loadEmbeddingPipeline();
|
|
104
115
|
const texts = request.texts.map((text) => applyEmbeddingPrefix(text, request.prefix, request.prefixes));
|
|
105
116
|
const tokenized = pipe.tokenizer(texts, {
|
|
@@ -119,13 +130,17 @@ async function handleMeasureEmbeddingInput(request) {
|
|
|
119
130
|
else {
|
|
120
131
|
counts = texts.map((text) => pipe.tokenizer.encode(text, { add_special_tokens: true }).length);
|
|
121
132
|
}
|
|
133
|
+
throwIfCancelled(request.id);
|
|
122
134
|
return { counts, maximum: embeddingMaximumTokens(pipe.tokenizer) };
|
|
123
135
|
}
|
|
124
136
|
async function handleRerank(request) {
|
|
137
|
+
throwIfCancelled(request.id);
|
|
125
138
|
const pipe = await loadRerankerPipeline(request.reranker);
|
|
126
139
|
const results = [];
|
|
127
140
|
for (const candidate of request.candidates) {
|
|
141
|
+
throwIfCancelled(request.id);
|
|
128
142
|
const output = await pipe({ text: request.query, text_pair: candidate.content });
|
|
143
|
+
throwIfCancelled(request.id);
|
|
129
144
|
const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
|
|
130
145
|
results.push({ chunkId: candidate.chunkId, score });
|
|
131
146
|
}
|
|
@@ -133,14 +148,35 @@ async function handleRerank(request) {
|
|
|
133
148
|
return results.slice(0, request.topK);
|
|
134
149
|
}
|
|
135
150
|
const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
|
|
151
|
+
function normalizeExtractedDocumentText(text) {
|
|
152
|
+
if (typeof text === "string") {
|
|
153
|
+
assertExtractedDocumentSize(text);
|
|
154
|
+
return text;
|
|
155
|
+
}
|
|
156
|
+
let bytes = 0;
|
|
157
|
+
for (const page of text) {
|
|
158
|
+
bytes += Buffer.byteLength(page);
|
|
159
|
+
if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
|
|
160
|
+
throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
bytes += Math.max(0, text.length - 1) * 2;
|
|
164
|
+
if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
|
|
165
|
+
throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
|
|
166
|
+
}
|
|
167
|
+
return text.join("\n\n");
|
|
168
|
+
}
|
|
136
169
|
async function handleExtractDocument(request) {
|
|
170
|
+
throwIfCancelled(request.id);
|
|
137
171
|
if (request.fileType === "pdf") {
|
|
138
172
|
const { extractText, getDocumentProxy } = await import("unpdf");
|
|
139
173
|
const bytes = await readFile(request.filePath);
|
|
174
|
+
throwIfCancelled(request.id);
|
|
140
175
|
const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
|
|
141
176
|
try {
|
|
142
177
|
const { text } = await extractText(pdf);
|
|
143
|
-
|
|
178
|
+
throwIfCancelled(request.id);
|
|
179
|
+
return { content: normalizeExtractedDocumentText(text), fileType: "pdf" };
|
|
144
180
|
}
|
|
145
181
|
finally {
|
|
146
182
|
await pdf.destroy();
|
|
@@ -148,6 +184,8 @@ async function handleExtractDocument(request) {
|
|
|
148
184
|
}
|
|
149
185
|
const mammoth = await import("mammoth");
|
|
150
186
|
const result = await mammoth.extractRawText({ path: request.filePath });
|
|
187
|
+
assertExtractedDocumentSize(result.value);
|
|
188
|
+
throwIfCancelled(request.id);
|
|
151
189
|
return { content: result.value, fileType: "docx" };
|
|
152
190
|
}
|
|
153
191
|
function isStringArray(value) {
|
|
@@ -219,11 +257,20 @@ function parseWorkerRequest(message) {
|
|
|
219
257
|
}
|
|
220
258
|
async function handleRequestMessage(message) {
|
|
221
259
|
let requestId = 0;
|
|
260
|
+
let cancelFrame = false;
|
|
222
261
|
try {
|
|
223
262
|
if (typeof message === "object" && message !== null && "id" in message && typeof message.id === "number") {
|
|
224
263
|
requestId = message.id;
|
|
225
264
|
}
|
|
265
|
+
if (typeof message === "object" && message !== null && "type" in message && message.type === "cancel") {
|
|
266
|
+
cancelFrame = true;
|
|
267
|
+
if (activeRequestIds.has(requestId))
|
|
268
|
+
cancelledRequestIds.add(requestId);
|
|
269
|
+
return undefined;
|
|
270
|
+
}
|
|
226
271
|
const request = parseWorkerRequest(message);
|
|
272
|
+
activeRequestIds.add(request.id);
|
|
273
|
+
throwIfCancelled(request.id);
|
|
227
274
|
const result = request.type === "embed"
|
|
228
275
|
? await handleEmbed(request)
|
|
229
276
|
: request.type === "measure_embedding_input"
|
|
@@ -231,20 +278,36 @@ async function handleRequestMessage(message) {
|
|
|
231
278
|
: request.type === "extract_document"
|
|
232
279
|
? await handleExtractDocument(request)
|
|
233
280
|
: await handleRerank(request);
|
|
281
|
+
throwIfCancelled(request.id);
|
|
234
282
|
return { id: request.id, result };
|
|
235
283
|
}
|
|
236
284
|
catch (error) {
|
|
237
285
|
return { id: requestId, error: error instanceof Error ? error.message : String(error) };
|
|
238
286
|
}
|
|
287
|
+
finally {
|
|
288
|
+
if (requestId !== 0 && !cancelFrame) {
|
|
289
|
+
activeRequestIds.delete(requestId);
|
|
290
|
+
cancelledRequestIds.delete(requestId);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
239
293
|
}
|
|
240
294
|
function writeStdioResponse(response) {
|
|
241
|
-
|
|
295
|
+
const line = `${JSON.stringify(response)}\n`;
|
|
296
|
+
if (Buffer.byteLength(line) <= MAX_WORKER_STDIO_FRAME_BYTES) {
|
|
297
|
+
process.stdout.write(line);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
process.stdout.write(`${JSON.stringify({
|
|
301
|
+
id: response.id,
|
|
302
|
+
error: `Model worker response exceeds the ${MAX_WORKER_STDIO_FRAME_BYTES}-byte frame limit`,
|
|
303
|
+
})}\n`);
|
|
242
304
|
}
|
|
243
305
|
function runIpcMode() {
|
|
244
306
|
process.on("message", (message) => {
|
|
245
307
|
void (async () => {
|
|
246
308
|
const response = await handleRequestMessage(message);
|
|
247
|
-
|
|
309
|
+
if (response)
|
|
310
|
+
process.send?.(response);
|
|
248
311
|
})();
|
|
249
312
|
});
|
|
250
313
|
}
|
|
@@ -258,9 +321,18 @@ function runStdioMode() {
|
|
|
258
321
|
redirectConsoleOutputToStderr();
|
|
259
322
|
const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
|
|
260
323
|
lines.on("line", (line) => {
|
|
324
|
+
if (Buffer.byteLength(line) > MAX_WORKER_STDIO_FRAME_BYTES) {
|
|
325
|
+
writeStdioResponse({
|
|
326
|
+
id: 0,
|
|
327
|
+
error: `Model worker request exceeds the ${MAX_WORKER_STDIO_FRAME_BYTES}-byte frame limit`,
|
|
328
|
+
});
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
261
331
|
void (async () => {
|
|
262
332
|
try {
|
|
263
|
-
|
|
333
|
+
const response = await handleRequestMessage(JSON.parse(line));
|
|
334
|
+
if (response)
|
|
335
|
+
writeStdioResponse(response);
|
|
264
336
|
}
|
|
265
337
|
catch (error) {
|
|
266
338
|
writeStdioResponse({ id: 0, error: error instanceof Error ? error.message : String(error) });
|