pi-knowledge 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +59 -22
  3. package/dist/index.d.ts +10 -4
  4. package/dist/index.js +145 -166
  5. package/dist/src/coverage-snapshot.d.ts +23 -0
  6. package/dist/src/coverage-snapshot.js +53 -0
  7. package/dist/src/daemon/client.d.ts +60 -0
  8. package/dist/src/daemon/client.js +360 -0
  9. package/dist/src/daemon/main.d.ts +2 -0
  10. package/dist/src/daemon/main.js +109 -0
  11. package/dist/src/daemon/protocol.d.ts +71 -0
  12. package/dist/src/daemon/protocol.js +170 -0
  13. package/dist/src/daemon/runtime.d.ts +26 -0
  14. package/dist/src/daemon/runtime.js +192 -0
  15. package/dist/src/daemon/server.d.ts +46 -0
  16. package/dist/src/daemon/server.js +500 -0
  17. package/dist/src/daemon/service.d.ts +49 -0
  18. package/dist/src/daemon/service.js +527 -0
  19. package/dist/src/diagnostics/health.d.ts +6 -0
  20. package/dist/src/diagnostics/health.js +135 -2
  21. package/dist/src/engine.d.ts +13 -8
  22. package/dist/src/engine.js +569 -100
  23. package/dist/src/filesystem-access.d.ts +9 -0
  24. package/dist/src/filesystem-access.js +100 -0
  25. package/dist/src/indexer/chunker.d.ts +10 -0
  26. package/dist/src/indexer/chunker.js +284 -9
  27. package/dist/src/mcp/http.d.ts +47 -0
  28. package/dist/src/mcp/http.js +564 -0
  29. package/dist/src/mcp/main.d.ts +2 -0
  30. package/dist/src/mcp/main.js +97 -0
  31. package/dist/src/mcp/protocol.d.ts +81 -0
  32. package/dist/src/mcp/protocol.js +444 -0
  33. package/dist/src/mcp/server.d.ts +42 -0
  34. package/dist/src/mcp/server.js +378 -0
  35. package/dist/src/model-worker-client.js +9 -0
  36. package/dist/src/model-worker-contract.d.ts +3 -0
  37. package/dist/src/model-worker-contract.js +8 -0
  38. package/dist/src/model-worker.js +76 -4
  39. package/dist/src/network/safe-fetch.d.ts +7 -0
  40. package/dist/src/network/safe-fetch.js +199 -0
  41. package/dist/src/node-runtime.d.ts +3 -0
  42. package/dist/src/node-runtime.js +78 -0
  43. package/dist/src/profiles.d.ts +15 -0
  44. package/dist/src/profiles.js +160 -0
  45. package/dist/src/runtime-config.js +5 -6
  46. package/dist/src/secure-files.d.ts +17 -0
  47. package/dist/src/secure-files.js +184 -0
  48. package/dist/src/status.js +12 -7
  49. package/dist/src/storage/sqlite.d.ts +4 -1
  50. package/dist/src/storage/sqlite.js +46 -3
  51. package/dist/src/type-guards.d.ts +1 -0
  52. package/dist/src/type-guards.js +3 -0
  53. package/dist/src/watcher/file-watcher.d.ts +3 -1
  54. package/dist/src/watcher/file-watcher.js +166 -32
  55. package/dist/src/windows-security.d.ts +4 -0
  56. package/dist/src/windows-security.js +47 -0
  57. package/docs/configuration.md +39 -9
  58. package/docs/daemon-profiles.md +302 -0
  59. package/docs/mcp-clients.md +266 -0
  60. package/docs/offline-mode.md +6 -6
  61. package/package.json +28 -2
@@ -0,0 +1,378 @@
1
+ import { existsSync, lstatSync, realpathSync } from "node:fs";
2
+ import { resolve } from "node:path";
3
+ import { assertOwnedDirectory } from "../secure-files.js";
4
+ import { isObjectRecord } from "../type-guards.js";
5
+ import { MAX_MCP_CONCURRENT_REQUESTS, MAX_MCP_INPUT_BYTES, MAX_MCP_OUTPUT_BYTES, MAX_MCP_OUTPUT_QUEUE_BYTES, MCP_SHUTDOWN_TIMEOUT_MS, McpProtocolError, McpProtocolSession, mcpErrorMessage, mcpRpcError, parseMcpRequest, } from "./protocol.js";
6
+ async function settlesWithin(promise, timeoutMs) {
7
+ let timer;
8
+ const timeout = new Promise((resolveTimeout) => {
9
+ timer = setTimeout(() => resolveTimeout(false), timeoutMs);
10
+ timer.unref();
11
+ });
12
+ const settled = promise.then(() => true, () => true);
13
+ try {
14
+ return await Promise.race([settled, timeout]);
15
+ }
16
+ finally {
17
+ if (timer)
18
+ clearTimeout(timer);
19
+ }
20
+ }
21
+ function parsePort(value) {
22
+ if (!/^\d+$/.test(value))
23
+ throw new Error("--port must be an integer from 0 to 65535");
24
+ const port = Number(value);
25
+ if (!Number.isSafeInteger(port) || port < 0 || port > 65_535) {
26
+ throw new Error("--port must be an integer from 0 to 65535");
27
+ }
28
+ return port;
29
+ }
30
+ function normalizeEndpoint(value) {
31
+ if (!value.startsWith("/") || value.startsWith("//") || /[?#\s]/.test(value)) {
32
+ throw new Error("--endpoint must be one absolute HTTP path without query or fragment");
33
+ }
34
+ return value;
35
+ }
36
+ function normalizeHost(value) {
37
+ if (value !== "127.0.0.1" && value !== "::1" && value !== "localhost") {
38
+ throw new Error("--host must be 127.0.0.1, ::1, or localhost");
39
+ }
40
+ return value;
41
+ }
42
+ function normalizeOrigin(value) {
43
+ let parsed;
44
+ try {
45
+ parsed = new URL(value);
46
+ }
47
+ catch {
48
+ throw new Error(`Invalid --allow-origin value: ${value}`);
49
+ }
50
+ if ((parsed.protocol !== "http:" && parsed.protocol !== "https:") ||
51
+ parsed.username ||
52
+ parsed.password ||
53
+ parsed.pathname !== "/" ||
54
+ parsed.search ||
55
+ parsed.hash) {
56
+ throw new Error(`--allow-origin must be an HTTP origin without path, credentials, query, or fragment: ${value}`);
57
+ }
58
+ return parsed.origin;
59
+ }
60
+ export function parseMcpArguments(argv) {
61
+ const options = {
62
+ allowRoots: [],
63
+ allowNetwork: false,
64
+ allowWrite: false,
65
+ allowAdmin: false,
66
+ transport: "stdio",
67
+ httpHost: "127.0.0.1",
68
+ httpPort: 3210,
69
+ httpEndpoint: "/mcp",
70
+ allowOrigins: [],
71
+ };
72
+ let transportSpecified = false;
73
+ let httpOptionSpecified = false;
74
+ for (let index = 0; index < argv.length; index++) {
75
+ const argument = argv[index];
76
+ if (argument === "--allow-network")
77
+ options.allowNetwork = true;
78
+ else if (argument === "--allow-write")
79
+ options.allowWrite = true;
80
+ else if (argument === "--allow-admin")
81
+ options.allowAdmin = true;
82
+ else if (argument === "--profile" ||
83
+ argument === "--allow-root" ||
84
+ argument === "--transport" ||
85
+ argument === "--host" ||
86
+ argument === "--port" ||
87
+ argument === "--endpoint" ||
88
+ argument === "--allow-origin") {
89
+ const value = argv[index + 1];
90
+ if (!value)
91
+ throw new Error(`Missing value for ${argument}`);
92
+ if (argument === "--profile") {
93
+ if (options.profileName !== undefined)
94
+ throw new Error("--profile can only be specified once");
95
+ options.profileName = value;
96
+ }
97
+ else if (argument === "--allow-root")
98
+ options.allowRoots.push(value);
99
+ else if (argument === "--transport") {
100
+ if (transportSpecified)
101
+ throw new Error("--transport can only be specified once");
102
+ if (value !== "stdio" && value !== "http")
103
+ throw new Error("--transport must be stdio or http");
104
+ options.transport = value;
105
+ transportSpecified = true;
106
+ }
107
+ else if (argument === "--host") {
108
+ options.httpHost = normalizeHost(value);
109
+ httpOptionSpecified = true;
110
+ }
111
+ else if (argument === "--port") {
112
+ options.httpPort = parsePort(value);
113
+ httpOptionSpecified = true;
114
+ }
115
+ else if (argument === "--endpoint") {
116
+ options.httpEndpoint = normalizeEndpoint(value);
117
+ httpOptionSpecified = true;
118
+ }
119
+ else {
120
+ options.allowOrigins.push(normalizeOrigin(value));
121
+ httpOptionSpecified = true;
122
+ }
123
+ index++;
124
+ }
125
+ else
126
+ throw new Error(`Unknown MCP option: ${argument}`);
127
+ }
128
+ if (options.transport === "stdio" && httpOptionSpecified) {
129
+ throw new Error("--host, --port, --endpoint, and --allow-origin require --transport http");
130
+ }
131
+ options.allowOrigins = [...new Set(options.allowOrigins)].sort();
132
+ return options;
133
+ }
134
+ export function canonicalizeAllowedRoots(paths) {
135
+ const roots = new Set();
136
+ for (const input of paths) {
137
+ const absolute = resolve(input);
138
+ if (!existsSync(absolute))
139
+ throw new Error(`Allowed root does not exist: ${absolute}`);
140
+ if (lstatSync(absolute).isSymbolicLink()) {
141
+ throw new Error(`Allowed root must be a canonical directory: ${absolute}`);
142
+ }
143
+ const canonical = realpathSync.native(absolute);
144
+ assertOwnedDirectory(canonical);
145
+ roots.add(canonical);
146
+ }
147
+ return [...roots].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
148
+ }
149
+ class OutputSink {
150
+ chain = Promise.resolve();
151
+ queuedBytes = 0;
152
+ fatalError;
153
+ output;
154
+ onFatal;
155
+ constructor(output, onFatal) {
156
+ this.output = output;
157
+ this.onFatal = onFatal;
158
+ output.on("error", (error) => this.fail(error));
159
+ output.on("close", () => this.fail(new Error("MCP output closed")));
160
+ }
161
+ fail(error) {
162
+ if (this.fatalError)
163
+ return;
164
+ this.fatalError = error;
165
+ this.onFatal(error);
166
+ }
167
+ writeLine(line) {
168
+ if (this.fatalError)
169
+ return Promise.reject(this.fatalError);
170
+ if (this.output.destroyed || this.output.writableEnded)
171
+ return Promise.reject(new Error("MCP output is closed"));
172
+ const { promise, resolve: resolveWrite, reject } = Promise.withResolvers();
173
+ let settled = false;
174
+ const finish = (error) => {
175
+ if (settled)
176
+ return;
177
+ settled = true;
178
+ this.output.off("drain", onDrain);
179
+ this.output.off("error", onError);
180
+ this.output.off("close", onClose);
181
+ if (error)
182
+ reject(error);
183
+ else
184
+ resolveWrite();
185
+ };
186
+ const onDrain = () => finish();
187
+ const onError = (error) => finish(error);
188
+ const onClose = () => finish(new Error("MCP output closed before draining"));
189
+ this.output.once("error", onError);
190
+ this.output.once("close", onClose);
191
+ if (this.output.write(line))
192
+ finish();
193
+ else
194
+ this.output.once("drain", onDrain);
195
+ return promise;
196
+ }
197
+ send(value) {
198
+ const line = `${JSON.stringify(value)}\n`;
199
+ const bytes = Buffer.byteLength(line);
200
+ if (bytes > MAX_MCP_OUTPUT_BYTES)
201
+ return Promise.reject(new Error("MCP response exceeds the output frame limit"));
202
+ if (this.fatalError)
203
+ return Promise.reject(this.fatalError);
204
+ if (this.queuedBytes + bytes > MAX_MCP_OUTPUT_QUEUE_BYTES) {
205
+ const error = new Error("MCP output queue is full");
206
+ this.fail(error);
207
+ this.output.destroy(error);
208
+ return Promise.reject(error);
209
+ }
210
+ this.queuedBytes += bytes;
211
+ const write = this.chain.then(() => this.writeLine(line));
212
+ this.chain = write
213
+ .catch((error) => this.fail(error instanceof Error ? error : new Error(String(error))))
214
+ .finally(() => {
215
+ this.queuedBytes -= bytes;
216
+ });
217
+ return write;
218
+ }
219
+ abort(error) {
220
+ this.fail(error);
221
+ if (!this.output.destroyed)
222
+ this.output.destroy(error);
223
+ }
224
+ async flush() {
225
+ await this.chain;
226
+ if (this.fatalError)
227
+ throw this.fatalError;
228
+ }
229
+ }
230
+ export class McpStdioAdapter {
231
+ output;
232
+ session;
233
+ active = new Set();
234
+ inputFrame = Buffer.allocUnsafe(MAX_MCP_INPUT_BYTES);
235
+ inputBytes = 0;
236
+ closing = false;
237
+ options;
238
+ overloadResponse;
239
+ constructor(options) {
240
+ this.options = options;
241
+ this.output = new OutputSink(options.output, (error) => {
242
+ this.closing = true;
243
+ options.input.destroy(error);
244
+ });
245
+ this.session = new McpProtocolSession({
246
+ ...options,
247
+ onError: (error) => this.logError(error),
248
+ });
249
+ }
250
+ async sendError(id, code, message) {
251
+ await this.output.send(mcpRpcError(id, code, message));
252
+ }
253
+ logError(error) {
254
+ this.options.error.write(`${mcpErrorMessage(error)}\n`);
255
+ }
256
+ async handleValue(value) {
257
+ let request;
258
+ try {
259
+ request = parseMcpRequest(value);
260
+ }
261
+ catch (error) {
262
+ const code = error instanceof McpProtocolError ? error.code : -32600;
263
+ await this.sendError(null, code, mcpErrorMessage(error));
264
+ return;
265
+ }
266
+ const handled = await this.session.handleRequest(request, (message) => this.output.send(message));
267
+ if (handled.response)
268
+ await this.output.send(handled.response);
269
+ }
270
+ rejectOverload(id, code, message) {
271
+ if (this.overloadResponse) {
272
+ this.closing = true;
273
+ this.options.input.destroy();
274
+ return;
275
+ }
276
+ const operation = this.sendError(id, code, message);
277
+ this.overloadResponse = operation;
278
+ this.active.add(operation);
279
+ void operation.then(() => {
280
+ this.active.delete(operation);
281
+ if (this.overloadResponse === operation)
282
+ this.overloadResponse = undefined;
283
+ }, (error) => {
284
+ this.active.delete(operation);
285
+ if (this.overloadResponse === operation)
286
+ this.overloadResponse = undefined;
287
+ this.logError(error);
288
+ });
289
+ }
290
+ acceptLine(line) {
291
+ let value;
292
+ try {
293
+ value = JSON.parse(line.toString("utf8"));
294
+ }
295
+ catch (error) {
296
+ if (this.active.size >= MAX_MCP_CONCURRENT_REQUESTS) {
297
+ this.rejectOverload(null, -32700, mcpErrorMessage(error));
298
+ return;
299
+ }
300
+ const operation = this.sendError(null, -32700, mcpErrorMessage(error));
301
+ this.active.add(operation);
302
+ void operation.then(() => this.active.delete(operation), (candidate) => {
303
+ this.active.delete(operation);
304
+ this.logError(candidate);
305
+ });
306
+ return;
307
+ }
308
+ if (isObjectRecord(value) &&
309
+ (value.method === "notifications/cancelled" || value.method === "notifications/initialized")) {
310
+ void this.handleValue(value).catch((error) => this.logError(error));
311
+ return;
312
+ }
313
+ if (this.active.size >= MAX_MCP_CONCURRENT_REQUESTS) {
314
+ const id = isObjectRecord(value) && "id" in value ? value.id : null;
315
+ const responseId = typeof id === "string" || typeof id === "number" ? id : null;
316
+ this.rejectOverload(responseId, -32000, "MCP adapter request limit reached");
317
+ return;
318
+ }
319
+ const operation = this.handleValue(value);
320
+ this.active.add(operation);
321
+ void operation.catch((error) => this.logError(error)).finally(() => this.active.delete(operation));
322
+ }
323
+ consumeChunk(chunk) {
324
+ let offset = 0;
325
+ while (offset < chunk.byteLength) {
326
+ const newline = chunk.indexOf(0x0a, offset);
327
+ const end = newline < 0 ? chunk.byteLength : newline;
328
+ const part = chunk.subarray(offset, end);
329
+ if (this.inputBytes + part.byteLength > MAX_MCP_INPUT_BYTES) {
330
+ throw new Error("MCP input frame exceeds the configured limit");
331
+ }
332
+ part.copy(this.inputFrame, this.inputBytes);
333
+ this.inputBytes += part.byteLength;
334
+ if (newline < 0)
335
+ return;
336
+ const line = this.inputFrame.subarray(0, this.inputBytes);
337
+ this.inputBytes = 0;
338
+ if (line.toString("utf8").trim())
339
+ this.acceptLine(line);
340
+ offset = newline + 1;
341
+ }
342
+ }
343
+ async run() {
344
+ let terminalError;
345
+ try {
346
+ for await (const chunk of this.options.input) {
347
+ if (this.closing)
348
+ break;
349
+ this.consumeChunk(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
350
+ }
351
+ const remainder = this.inputFrame.subarray(0, this.inputBytes);
352
+ if (remainder.toString("utf8").trim())
353
+ this.acceptLine(remainder);
354
+ }
355
+ catch (error) {
356
+ terminalError = error;
357
+ }
358
+ this.closing = true;
359
+ this.session.abortAll();
360
+ const cleanup = Promise.allSettled([...this.active]).then(() => this.output.flush());
361
+ if (!(await settlesWithin(cleanup, MCP_SHUTDOWN_TIMEOUT_MS))) {
362
+ const error = new Error("MCP shutdown timed out while draining output");
363
+ this.output.abort(error);
364
+ void cleanup.catch(() => undefined);
365
+ terminalError ??= error;
366
+ }
367
+ else {
368
+ try {
369
+ await cleanup;
370
+ }
371
+ catch (error) {
372
+ terminalError ??= error;
373
+ }
374
+ }
375
+ if (terminalError !== undefined)
376
+ throw terminalError;
377
+ }
378
+ }
@@ -3,6 +3,7 @@ import { existsSync, readdirSync, statSync } from "node:fs";
3
3
  import { basename, delimiter, join } from "node:path";
4
4
  import { StringDecoder } from "node:string_decoder";
5
5
  import { fileURLToPath } from "node:url";
6
+ import { assertExtractedDocumentSize, MAX_WORKER_STDIO_FRAME_BYTES } from "./model-worker-contract.js";
6
7
  import { getConfiguredNodePath, writeRuntimeConfig } from "./runtime-config.js";
7
8
  let worker = null;
8
9
  let nextRequestId = 1;
@@ -229,6 +230,7 @@ class ModelWorkerTransport {
229
230
  };
230
231
  if (signal) {
231
232
  abortHandler = () => {
233
+ this.writeRequest({ id, type: "cancel" }, () => { });
232
234
  this.pending.delete(id);
233
235
  cleanup();
234
236
  reject(new Error("Cancelled"));
@@ -355,6 +357,12 @@ class StdioModelWorkerTransport extends ModelWorkerTransport {
355
357
  });
356
358
  }
357
359
  handleStdoutChunk(chunk) {
360
+ if (Buffer.byteLength(this.stdoutBuffer) + chunk.byteLength > MAX_WORKER_STDIO_FRAME_BYTES) {
361
+ this.rejectPending(new Error("Model worker stdio response exceeds the configured frame limit"));
362
+ if (!this.child.killed)
363
+ this.child.kill("SIGKILL");
364
+ return;
365
+ }
358
366
  this.stdoutBuffer += this.decoder.write(chunk);
359
367
  let newlineIndex = this.stdoutBuffer.indexOf("\n");
360
368
  while (newlineIndex >= 0) {
@@ -423,6 +431,7 @@ export async function extractDocumentInModelWorker(filePath, fileType, signal) {
423
431
  result.fileType !== fileType) {
424
432
  throw new Error("Invalid document extraction worker response");
425
433
  }
434
+ assertExtractedDocumentSize(result.content);
426
435
  return { content: result.content, fileType };
427
436
  }
428
437
  export async function rerankInModelWorker(query, candidates, topK, reranker, signal) {
@@ -0,0 +1,3 @@
1
+ export declare const MAX_EXTRACTED_DOCUMENT_BYTES: number;
2
+ export declare const MAX_WORKER_STDIO_FRAME_BYTES: number;
3
+ export declare function assertExtractedDocumentSize(content: string): void;
@@ -0,0 +1,8 @@
1
+ export const MAX_EXTRACTED_DOCUMENT_BYTES = 8 * 1024 * 1024;
2
+ export const MAX_WORKER_STDIO_FRAME_BYTES = 64 * 1024 * 1024;
3
+ export function assertExtractedDocumentSize(content) {
4
+ const bytes = Buffer.byteLength(content);
5
+ if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
6
+ throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
7
+ }
8
+ }
@@ -2,12 +2,19 @@ import { readFile } from "node:fs/promises";
2
2
  import { join } from "node:path";
3
3
  import { createInterface } from "node:readline";
4
4
  import { decorateEmbeddingInput, E5_INSTRUCTIONS, NO_INSTRUCTIONS, } from "./embedding/instructions.js";
5
+ import { assertExtractedDocumentSize, MAX_EXTRACTED_DOCUMENT_BYTES, MAX_WORKER_STDIO_FRAME_BYTES, } from "./model-worker-contract.js";
5
6
  import { DEFAULT_RERANKER_REMOTE_HOST, DEFAULT_RERANKER_REMOTE_PATH_TEMPLATE, isHfRerankerConfig, rerankerCacheKey, resolveRerankerConfig, } from "./search/reranker-config.js";
6
7
  import { rerankWithSingleRawLogit } from "./search/reranker-logits.js";
7
8
  import { getDefaultKnowledgeDir } from "./storage/sqlite.js";
8
9
  let embeddingPipeline = null;
9
10
  let embeddingPipelinePromise = null;
10
11
  let rerankerPipeline = null;
12
+ const cancelledRequestIds = new Set();
13
+ const activeRequestIds = new Set();
14
+ function throwIfCancelled(id) {
15
+ if (cancelledRequestIds.has(id))
16
+ throw new Error("Cancelled");
17
+ }
11
18
  function getModelCacheDir() {
12
19
  return process.env.PI_KNOWLEDGE_MODEL_CACHE_DIR ?? join(getDefaultKnowledgeDir(), "models");
13
20
  }
@@ -82,13 +89,16 @@ function applyEmbeddingPrefix(text, role, prefixes) {
82
89
  return decorateEmbeddingInput(text, role, prefixes === "on" ? E5_INSTRUCTIONS : NO_INSTRUCTIONS);
83
90
  }
84
91
  async function handleEmbed(request) {
92
+ throwIfCancelled(request.id);
85
93
  const pipe = await loadEmbeddingPipeline();
86
94
  const vectors = [];
87
95
  for (const text of request.texts) {
96
+ throwIfCancelled(request.id);
88
97
  const output = await pipe(applyEmbeddingPrefix(text, request.prefix, request.prefixes), {
89
98
  pooling: "mean",
90
99
  normalize: true,
91
100
  });
101
+ throwIfCancelled(request.id);
92
102
  vectors.push(Array.from(output.data));
93
103
  }
94
104
  return vectors;
@@ -100,6 +110,7 @@ function embeddingMaximumTokens(tokenizer) {
100
110
  : 512;
101
111
  }
102
112
  async function handleMeasureEmbeddingInput(request) {
113
+ throwIfCancelled(request.id);
103
114
  const pipe = await loadEmbeddingPipeline();
104
115
  const texts = request.texts.map((text) => applyEmbeddingPrefix(text, request.prefix, request.prefixes));
105
116
  const tokenized = pipe.tokenizer(texts, {
@@ -119,13 +130,17 @@ async function handleMeasureEmbeddingInput(request) {
119
130
  else {
120
131
  counts = texts.map((text) => pipe.tokenizer.encode(text, { add_special_tokens: true }).length);
121
132
  }
133
+ throwIfCancelled(request.id);
122
134
  return { counts, maximum: embeddingMaximumTokens(pipe.tokenizer) };
123
135
  }
124
136
  async function handleRerank(request) {
137
+ throwIfCancelled(request.id);
125
138
  const pipe = await loadRerankerPipeline(request.reranker);
126
139
  const results = [];
127
140
  for (const candidate of request.candidates) {
141
+ throwIfCancelled(request.id);
128
142
  const output = await pipe({ text: request.query, text_pair: candidate.content });
143
+ throwIfCancelled(request.id);
129
144
  const score = Array.isArray(output) ? (output[0]?.score ?? 0) : (output?.score ?? 0);
130
145
  results.push({ chunkId: candidate.chunkId, score });
131
146
  }
@@ -133,14 +148,35 @@ async function handleRerank(request) {
133
148
  return results.slice(0, request.topK);
134
149
  }
135
150
  const PDFJS_ERRORS_ONLY_VERBOSITY = 0;
151
+ function normalizeExtractedDocumentText(text) {
152
+ if (typeof text === "string") {
153
+ assertExtractedDocumentSize(text);
154
+ return text;
155
+ }
156
+ let bytes = 0;
157
+ for (const page of text) {
158
+ bytes += Buffer.byteLength(page);
159
+ if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
160
+ throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
161
+ }
162
+ }
163
+ bytes += Math.max(0, text.length - 1) * 2;
164
+ if (bytes > MAX_EXTRACTED_DOCUMENT_BYTES) {
165
+ throw new Error(`Extracted document exceeds the ${MAX_EXTRACTED_DOCUMENT_BYTES}-byte text limit`);
166
+ }
167
+ return text.join("\n\n");
168
+ }
136
169
  async function handleExtractDocument(request) {
170
+ throwIfCancelled(request.id);
137
171
  if (request.fileType === "pdf") {
138
172
  const { extractText, getDocumentProxy } = await import("unpdf");
139
173
  const bytes = await readFile(request.filePath);
174
+ throwIfCancelled(request.id);
140
175
  const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: PDFJS_ERRORS_ONLY_VERBOSITY });
141
176
  try {
142
177
  const { text } = await extractText(pdf);
143
- return { content: Array.isArray(text) ? text.join("\n\n") : text, fileType: "pdf" };
178
+ throwIfCancelled(request.id);
179
+ return { content: normalizeExtractedDocumentText(text), fileType: "pdf" };
144
180
  }
145
181
  finally {
146
182
  await pdf.destroy();
@@ -148,6 +184,8 @@ async function handleExtractDocument(request) {
148
184
  }
149
185
  const mammoth = await import("mammoth");
150
186
  const result = await mammoth.extractRawText({ path: request.filePath });
187
+ assertExtractedDocumentSize(result.value);
188
+ throwIfCancelled(request.id);
151
189
  return { content: result.value, fileType: "docx" };
152
190
  }
153
191
  function isStringArray(value) {
@@ -219,11 +257,20 @@ function parseWorkerRequest(message) {
219
257
  }
220
258
  async function handleRequestMessage(message) {
221
259
  let requestId = 0;
260
+ let cancelFrame = false;
222
261
  try {
223
262
  if (typeof message === "object" && message !== null && "id" in message && typeof message.id === "number") {
224
263
  requestId = message.id;
225
264
  }
265
+ if (typeof message === "object" && message !== null && "type" in message && message.type === "cancel") {
266
+ cancelFrame = true;
267
+ if (activeRequestIds.has(requestId))
268
+ cancelledRequestIds.add(requestId);
269
+ return undefined;
270
+ }
226
271
  const request = parseWorkerRequest(message);
272
+ activeRequestIds.add(request.id);
273
+ throwIfCancelled(request.id);
227
274
  const result = request.type === "embed"
228
275
  ? await handleEmbed(request)
229
276
  : request.type === "measure_embedding_input"
@@ -231,20 +278,36 @@ async function handleRequestMessage(message) {
231
278
  : request.type === "extract_document"
232
279
  ? await handleExtractDocument(request)
233
280
  : await handleRerank(request);
281
+ throwIfCancelled(request.id);
234
282
  return { id: request.id, result };
235
283
  }
236
284
  catch (error) {
237
285
  return { id: requestId, error: error instanceof Error ? error.message : String(error) };
238
286
  }
287
+ finally {
288
+ if (requestId !== 0 && !cancelFrame) {
289
+ activeRequestIds.delete(requestId);
290
+ cancelledRequestIds.delete(requestId);
291
+ }
292
+ }
239
293
  }
240
294
  function writeStdioResponse(response) {
241
- process.stdout.write(`${JSON.stringify(response)}\n`);
295
+ const line = `${JSON.stringify(response)}\n`;
296
+ if (Buffer.byteLength(line) <= MAX_WORKER_STDIO_FRAME_BYTES) {
297
+ process.stdout.write(line);
298
+ return;
299
+ }
300
+ process.stdout.write(`${JSON.stringify({
301
+ id: response.id,
302
+ error: `Model worker response exceeds the ${MAX_WORKER_STDIO_FRAME_BYTES}-byte frame limit`,
303
+ })}\n`);
242
304
  }
243
305
  function runIpcMode() {
244
306
  process.on("message", (message) => {
245
307
  void (async () => {
246
308
  const response = await handleRequestMessage(message);
247
- process.send?.(response);
309
+ if (response)
310
+ process.send?.(response);
248
311
  })();
249
312
  });
250
313
  }
@@ -258,9 +321,18 @@ function runStdioMode() {
258
321
  redirectConsoleOutputToStderr();
259
322
  const lines = createInterface({ input: process.stdin, crlfDelay: Infinity });
260
323
  lines.on("line", (line) => {
324
+ if (Buffer.byteLength(line) > MAX_WORKER_STDIO_FRAME_BYTES) {
325
+ writeStdioResponse({
326
+ id: 0,
327
+ error: `Model worker request exceeds the ${MAX_WORKER_STDIO_FRAME_BYTES}-byte frame limit`,
328
+ });
329
+ return;
330
+ }
261
331
  void (async () => {
262
332
  try {
263
- writeStdioResponse(await handleRequestMessage(JSON.parse(line)));
333
+ const response = await handleRequestMessage(JSON.parse(line));
334
+ if (response)
335
+ writeStdioResponse(response);
264
336
  }
265
337
  catch (error) {
266
338
  writeStdioResponse({ id: 0, error: error instanceof Error ? error.message : String(error) });
@@ -0,0 +1,7 @@
1
+ interface SafeFetchOptions {
2
+ maxBytes?: number;
3
+ timeoutMs?: number;
4
+ maxRedirects?: number;
5
+ }
6
+ export declare function fetchPublicText(source: string, signal?: AbortSignal, options?: SafeFetchOptions): Promise<string>;
7
+ export {};