pi-multimodal-proxy 1.6.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +242 -220
- package/extensions/__tests__/integration.test.ts +0 -2
- package/extensions/__tests__/internal.test.ts +326 -12
- package/extensions/internal.ts +465 -28
- package/extensions/vision-proxy.ts +2493 -2326
- package/package.json +2 -2
- package/.pi/ghost-autocomplete/metrics.jsonl +0 -192
- package/.pi/ghost-autocomplete/profile.jsonl +0 -19
- package/bash.exe.stackdump +0 -28
package/extensions/internal.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { createHash } from "node:crypto";
|
|
7
|
-
import { mkdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
7
|
+
import { lstat, mkdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
8
8
|
import os from "node:os";
|
|
9
9
|
import { basename, dirname, extname, join, parse, relative } from "node:path";
|
|
10
10
|
import type { ImageContent as PiAiImage } from "@earendil-works/pi-ai";
|
|
@@ -55,22 +55,162 @@ export interface ImageMeta {
|
|
|
55
55
|
filename?: string; // basename only
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
/**
|
|
59
|
-
|
|
58
|
+
/**
|
|
59
|
+
* In-memory map: image hash → dimensions + filename, populated on first
|
|
60
|
+
* ingestion. Held per session (see SessionState in vision-proxy) rather than as
|
|
61
|
+
* a process-global, so forked/resumed sessions never inherit stale metadata.
|
|
62
|
+
*/
|
|
63
|
+
export type ImageMetaStore = Map<string, ImageMeta>;
|
|
64
|
+
|
|
65
|
+
/** Create an empty per-session image-metadata store. */
|
|
66
|
+
export function createImageMetaStore(): ImageMetaStore {
|
|
67
|
+
return new Map<string, ImageMeta>();
|
|
68
|
+
}
|
|
60
69
|
|
|
61
70
|
/** Maximum pixel dimension for decoded images. Prevents decode bombs (e.g., 10 MB PNG → 500 MB bitmap). */
|
|
62
71
|
const MAX_IMAGE_DIMENSION = 16384; // 16K × 16K ≈ 1 billion pixels max
|
|
63
72
|
|
|
64
|
-
/** Maximum entries
|
|
73
|
+
/** Maximum entries per image-metadata store to prevent unbounded memory growth. */
|
|
65
74
|
const IMAGE_META_MAX = 500;
|
|
66
75
|
|
|
67
|
-
function evictImageMeta(): void {
|
|
68
|
-
while (
|
|
69
|
-
const first =
|
|
70
|
-
if (first !== undefined)
|
|
76
|
+
function evictImageMeta(meta: ImageMetaStore): void {
|
|
77
|
+
while (meta.size > IMAGE_META_MAX) {
|
|
78
|
+
const first = meta.keys().next().value;
|
|
79
|
+
if (first !== undefined) meta.delete(first);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ── Session image recall ────────────────────────────────────────────────────
|
|
84
|
+
//
|
|
85
|
+
// Retains the actual image bytes (base64) of images seen this session, keyed by
|
|
86
|
+
// hash, so the agent can re-query a previously-seen image with analyze_image
|
|
87
|
+
// even when it is no longer attached to the current turn (e.g. a screenshot the
|
|
88
|
+
// user pasted several turns ago). Storage is in-memory only — image bytes are
|
|
89
|
+
// never written to the session log or disk, keeping the existing data-egress
|
|
90
|
+
// posture intact. Insertion order is used for LRU eviction once the byte budget
|
|
91
|
+
// is exceeded.
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Per-session retained-bytes store for image recall: hash → base64 bytes + mime
|
|
95
|
+
* type, plus a running total of decoded bytes for budget enforcement. Held per
|
|
96
|
+
* session (see SessionState in vision-proxy) so retained image bytes never leak
|
|
97
|
+
* across sessions, mirroring the per-session image-metadata store.
|
|
98
|
+
*/
|
|
99
|
+
export interface ImageDataStore {
|
|
100
|
+
map: Map<string, { data: string; mimeType: string }>;
|
|
101
|
+
totalBytes: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Create an empty per-session image-recall byte store. */
|
|
105
|
+
export function createImageDataStore(): ImageDataStore {
|
|
106
|
+
return { map: new Map(), totalBytes: 0 };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Default byte budget for retained image data (decoded bytes, ≈64 MB). */
|
|
110
|
+
const IMAGE_DATA_MAX_BYTES_DEFAULT = 64 * 1024 * 1024;
|
|
111
|
+
|
|
112
|
+
/** Resolve the recall byte budget, allowing an env override. */
|
|
113
|
+
function imageDataMaxBytes(): number {
|
|
114
|
+
const raw = process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
115
|
+
if (raw) {
|
|
116
|
+
const n = Number.parseInt(raw, 10);
|
|
117
|
+
if (Number.isFinite(n) && n >= 0) return n;
|
|
118
|
+
}
|
|
119
|
+
return IMAGE_DATA_MAX_BYTES_DEFAULT;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function evictImageData(store: ImageDataStore): void {
|
|
123
|
+
const budget = imageDataMaxBytes();
|
|
124
|
+
// When budget is 0, allow full eviction (recall disabled).
|
|
125
|
+
// Otherwise keep at least one entry so an oversized image is still recallable.
|
|
126
|
+
const minRetained = budget === 0 ? 0 : 1;
|
|
127
|
+
while (store.totalBytes > budget && store.map.size > minRetained) {
|
|
128
|
+
const first = store.map.keys().next().value;
|
|
129
|
+
if (first === undefined) break;
|
|
130
|
+
const v = store.map.get(first);
|
|
131
|
+
store.map.delete(first);
|
|
132
|
+
if (v) store.totalBytes -= Buffer.byteLength(v.data, "base64");
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Retain an image's bytes for later recall. No-op if already retained (LRU bumped). */
|
|
137
|
+
export function storeImageData(store: ImageDataStore, hash: string, data: string, mimeType: string): void {
|
|
138
|
+
if (!hash || !data) return;
|
|
139
|
+
const existing = store.map.get(hash);
|
|
140
|
+
if (existing) {
|
|
141
|
+
// Bump recency: re-insert at the end of the iteration order.
|
|
142
|
+
store.map.delete(hash);
|
|
143
|
+
store.map.set(hash, existing);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
store.map.set(hash, { data, mimeType });
|
|
147
|
+
store.totalBytes += Buffer.byteLength(data, "base64");
|
|
148
|
+
evictImageData(store);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Fetch retained image bytes by hash, bumping recency. Undefined if not retained. */
|
|
152
|
+
export function getImageData(store: ImageDataStore, hash: string): { data: string; mimeType: string } | undefined {
|
|
153
|
+
const v = store.map.get(hash);
|
|
154
|
+
if (v) {
|
|
155
|
+
store.map.delete(hash);
|
|
156
|
+
store.map.set(hash, v);
|
|
71
157
|
}
|
|
158
|
+
return v;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Test/maintenance helper: drop all retained image bytes. */
|
|
162
|
+
export function clearImageData(store: ImageDataStore): void {
|
|
163
|
+
store.map.clear();
|
|
164
|
+
store.totalBytes = 0;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Parse an analyze_image reference as a session-recall handle.
|
|
169
|
+
*
|
|
170
|
+
* Accepts the `image="..."` value carried by <vision_proxy_description> and
|
|
171
|
+
* related fences — either a bare hash, a `sha256:`-prefixed hash, or a hash with
|
|
172
|
+
* a `#crop:...` suffix (the crop suffix is ignored; recall returns the full
|
|
173
|
+
* image and any crop is re-applied via the tool's crop argument). Returns the
|
|
174
|
+
* normalized lowercase hash, or null if the reference is not a recall handle
|
|
175
|
+
* (in which case it should be treated as a file path).
|
|
176
|
+
*/
|
|
177
|
+
export function parseRecallRef(ref: string): string | null {
|
|
178
|
+
let s = ref.trim();
|
|
179
|
+
if (s.startsWith("sha256:")) s = s.slice("sha256:".length);
|
|
180
|
+
const hashPart = s.split("#")[0];
|
|
181
|
+
const re = new RegExp(`^[a-f0-9]{${HASH_HEX_LEN}}$`, "i");
|
|
182
|
+
return re.test(hashPart) ? hashPart.toLowerCase() : null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// ── Live progress indicator ─────────────────────────────────────────────────
|
|
186
|
+
|
|
187
|
+
/** Braille spinner frames used by the live status indicator during slow calls. */
|
|
188
|
+
export const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
189
|
+
|
|
190
|
+
/** Pick a spinner frame for a given tick (wraps around). */
|
|
191
|
+
export function spinnerFrame(tick: number): string {
|
|
192
|
+
const n = SPINNER_FRAMES.length;
|
|
193
|
+
const i = ((Math.trunc(tick) % n) + n) % n;
|
|
194
|
+
return SPINNER_FRAMES[i];
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Format the status-line text shown while a vision/video call is in flight. */
|
|
198
|
+
export function formatProgressStatus(label: string, frame: string, elapsedSec: number): string {
|
|
199
|
+
const secs = Math.max(0, Math.trunc(elapsedSec));
|
|
200
|
+
return `multimodal-proxy ${frame} ${label} (${secs}s)`;
|
|
72
201
|
}
|
|
73
202
|
|
|
203
|
+
// ── Recall affordance ───────────────────────────────────────────────────────
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Persistent reminder injected once per turn alongside recalled image
|
|
207
|
+
* descriptions, restating that earlier images can be re-queried by id. This is
|
|
208
|
+
* trusted extension text (not image-derived), so it is placed outside the
|
|
209
|
+
* untrusted description fence.
|
|
210
|
+
*/
|
|
211
|
+
export const RECALL_HINT =
|
|
212
|
+
'You can re-examine or crop this or any earlier image at any time by calling analyze_image with its image id (the image="…" value above) — no re-attachment or file path needed.';
|
|
213
|
+
|
|
74
214
|
// ── Crop types ────────────────────────────────────────────────────────────
|
|
75
215
|
|
|
76
216
|
export type NamedRegion =
|
|
@@ -919,16 +1059,33 @@ function driveAccessDisabled(): boolean {
|
|
|
919
1059
|
|
|
920
1060
|
/**
|
|
921
1061
|
* Check that a resolved file path is within a safe directory.
|
|
922
|
-
* By default allows tmpdir, cwd, and local Windows drive paths;
|
|
923
|
-
* on non-drive platforms via PI_VISION_PROXY_ALLOW_HOME=1.
|
|
1062
|
+
* By default allows tmpdir, /tmp (system-wide Unix temp), cwd, and local Windows drive paths;
|
|
1063
|
+
* opt into homedir on non-drive platforms via PI_VISION_PROXY_ALLOW_HOME=1.
|
|
924
1064
|
* Both sides are canonicalized via realpath to handle symlinks and Windows 8.3 short names.
|
|
1065
|
+
* If the target file does not exist, the parent directory is resolved so callers can
|
|
1066
|
+
* distinguish "allowed dir but missing file" (→ "unreadable") from a genuinely denied path.
|
|
925
1067
|
*/
|
|
926
1068
|
export async function isPathAllowed(filePath: string): Promise<boolean> {
|
|
927
1069
|
let resolved: string;
|
|
928
1070
|
try {
|
|
929
1071
|
resolved = (await realpath(filePath)).toLowerCase();
|
|
930
1072
|
} catch {
|
|
931
|
-
|
|
1073
|
+
// realpath failed. Determine whether the path itself exists (e.g. broken symlink)
|
|
1074
|
+
// or is simply absent. For broken symlinks the target is outside our control, so
|
|
1075
|
+
// deny. For absent paths, fall back to the parent directory so that
|
|
1076
|
+
// readImageFileWithReason can return "unreadable" rather than the misleading "denied".
|
|
1077
|
+
try {
|
|
1078
|
+
await lstat(filePath); // succeeds for broken symlinks; throws for absent paths
|
|
1079
|
+
return false; // path exists (broken symlink or inaccessible) — deny
|
|
1080
|
+
} catch {
|
|
1081
|
+
// Path is absent — resolve via parent to check if it would be in an allowed dir.
|
|
1082
|
+
}
|
|
1083
|
+
const parent = dirname(filePath);
|
|
1084
|
+
try {
|
|
1085
|
+
resolved = join((await realpath(parent)).toLowerCase(), basename(filePath).toLowerCase());
|
|
1086
|
+
} catch {
|
|
1087
|
+
return false;
|
|
1088
|
+
}
|
|
932
1089
|
}
|
|
933
1090
|
|
|
934
1091
|
const tmp = await canonical(os.tmpdir?.() ?? "/tmp");
|
|
@@ -937,6 +1094,14 @@ export async function isPathAllowed(filePath: string): Promise<boolean> {
|
|
|
937
1094
|
if (tmp && isInsideOrSame(resolved, tmp)) return true;
|
|
938
1095
|
if (cwd && isInsideOrSame(resolved, cwd)) return true;
|
|
939
1096
|
|
|
1097
|
+
// On Unix, /tmp (the POSIX system-wide temp dir) may differ from os.tmpdir()
|
|
1098
|
+
// (e.g. on macOS where os.tmpdir() returns a per-user dir like /var/folders/…/T).
|
|
1099
|
+
// Allow it explicitly so files written to /tmp are always accessible.
|
|
1100
|
+
if (os.platform() !== "win32") {
|
|
1101
|
+
const unixTmp = await canonical("/tmp");
|
|
1102
|
+
if (unixTmp && unixTmp !== tmp && isInsideOrSame(resolved, unixTmp)) return true;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
940
1105
|
if (process.env.PI_VISION_PROXY_ALLOW_HOME === "1") {
|
|
941
1106
|
const home = await canonical(os.homedir?.());
|
|
942
1107
|
if (home && isInsideOrSame(resolved, home)) return true;
|
|
@@ -960,6 +1125,11 @@ export async function readImageFileWithReason(filePath: string): Promise<ReadIma
|
|
|
960
1125
|
} catch {
|
|
961
1126
|
return { image: null, reason: "unreadable" };
|
|
962
1127
|
}
|
|
1128
|
+
// Post-read re-verification: the initial isPathAllowed() may have passed via the
|
|
1129
|
+
// parent-dir fallback when the file did not yet exist. A symlink could have been
|
|
1130
|
+
// swapped in during that window. Now that the file exists, realpath() resolves it
|
|
1131
|
+
// fully — catching any symlink pointing outside the allow-list (TOCTOU mitigation).
|
|
1132
|
+
if (!(await isPathAllowed(filePath))) return { image: null, reason: "denied" };
|
|
963
1133
|
if (content.length === 0) return { image: null, reason: "empty", bytes: 0 };
|
|
964
1134
|
const limit = maxImageFileBytes();
|
|
965
1135
|
if (content.length > limit) return { image: null, reason: "too-large", bytes: content.length };
|
|
@@ -1194,8 +1364,8 @@ function safeDimensions(data: Buffer): { width: number; height: number } | undef
|
|
|
1194
1364
|
return dims;
|
|
1195
1365
|
}
|
|
1196
1366
|
|
|
1197
|
-
export function storeImageMeta(hash: string, imageBufferOrData: Buffer | string, filename?: string): void {
|
|
1198
|
-
const existing =
|
|
1367
|
+
export function storeImageMeta(meta: ImageMetaStore, hash: string, imageBufferOrData: Buffer | string, filename?: string): void {
|
|
1368
|
+
const existing = meta.get(hash);
|
|
1199
1369
|
if (existing) {
|
|
1200
1370
|
// Backfill filename if previously stored without one
|
|
1201
1371
|
if (filename && !existing.filename) {
|
|
@@ -1217,8 +1387,8 @@ export function storeImageMeta(hash: string, imageBufferOrData: Buffer | string,
|
|
|
1217
1387
|
}
|
|
1218
1388
|
const dims = safeDimensions(buf);
|
|
1219
1389
|
if (dims) {
|
|
1220
|
-
|
|
1221
|
-
evictImageMeta();
|
|
1390
|
+
meta.set(hash, { width: dims.width, height: dims.height, filename });
|
|
1391
|
+
evictImageMeta(meta);
|
|
1222
1392
|
}
|
|
1223
1393
|
}
|
|
1224
1394
|
|
|
@@ -1342,10 +1512,285 @@ export function cropSignature(crop: ResolvedCrop): string {
|
|
|
1342
1512
|
/** Whether ImageScript is available for cropping. */
|
|
1343
1513
|
export const hasCropper = true;
|
|
1344
1514
|
|
|
1515
|
+
/**
|
|
1516
|
+
* Wall-clock limit for a single image decode, in milliseconds. Override via env
|
|
1517
|
+
* for slow hosts or very large legitimate images.
|
|
1518
|
+
*/
|
|
1519
|
+
function decodeTimeoutMs(): number {
|
|
1520
|
+
const raw = process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS;
|
|
1521
|
+
if (raw) {
|
|
1522
|
+
const n = Number.parseInt(raw, 10);
|
|
1523
|
+
if (Number.isFinite(n) && n > 0) return n;
|
|
1524
|
+
}
|
|
1525
|
+
return 5000;
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
/**
|
|
1529
|
+
* Decode image bytes, rejecting if the decoder does not settle within the
|
|
1530
|
+
* timeout.
|
|
1531
|
+
*
|
|
1532
|
+
* SCOPE / LIMITATION: ImageScript's codecs are synchronous WASM. Once the WASM
|
|
1533
|
+
* `decode()` call starts it blocks the single Node thread until it returns, so
|
|
1534
|
+
* this timer cannot pre-empt a decode that is genuinely spinning on a crafted
|
|
1535
|
+
* body — the timeout callback can't run while the event loop is blocked. What
|
|
1536
|
+
* this wrapper *does* bound is the portions that yield (first-call WASM
|
|
1537
|
+
* instantiation and any async codec paths) and it stops a late-resolving decode
|
|
1538
|
+
* from leaving the caller hanging forever. The primary defence against
|
|
1539
|
+
* pathological inputs remains the dimension pre-check in cropImage(); full CPU
|
|
1540
|
+
* isolation would require running the decode in a terminable worker thread.
|
|
1541
|
+
*/
|
|
1542
|
+
async function decodeWithTimeout(imageBytes: Buffer): Promise<Image> {
|
|
1543
|
+
const timeoutMs = decodeTimeoutMs();
|
|
1544
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1545
|
+
const timeout = new Promise<never>((_resolve, reject) => {
|
|
1546
|
+
timer = setTimeout(() => reject(new Error(`Image.decode exceeded ${timeoutMs}ms timeout`)), timeoutMs);
|
|
1547
|
+
});
|
|
1548
|
+
try {
|
|
1549
|
+
return await Promise.race([Image.decode(new Uint8Array(imageBytes)), timeout]);
|
|
1550
|
+
} finally {
|
|
1551
|
+
if (timer) clearTimeout(timer);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
/**
|
|
1556
|
+
* In-thread decode → crop → encode. Bounded only by decodeWithTimeout, which
|
|
1557
|
+
* cannot pre-empt a synchronous WASM hang (see its doc). Used as a fallback when
|
|
1558
|
+
* the worker path is unavailable or disabled.
|
|
1559
|
+
*/
|
|
1560
|
+
async function cropInThread(
|
|
1561
|
+
imageBytes: Buffer,
|
|
1562
|
+
crop: ResolvedCrop,
|
|
1563
|
+
mimeType?: string,
|
|
1564
|
+
): Promise<Buffer | null> {
|
|
1565
|
+
const img = await decodeWithTimeout(imageBytes);
|
|
1566
|
+
// Double-check decoded dimensions (image-size is header-only, actual may differ)
|
|
1567
|
+
if (img.width > MAX_IMAGE_DIMENSION || img.height > MAX_IMAGE_DIMENSION) {
|
|
1568
|
+
return null;
|
|
1569
|
+
}
|
|
1570
|
+
const cropped = img.crop(crop.x, crop.y, crop.width, crop.height);
|
|
1571
|
+
let encoded: Uint8Array;
|
|
1572
|
+
if (mimeType === "image/png") {
|
|
1573
|
+
encoded = await cropped.encode(1); // PNG with compression level 1 (fast)
|
|
1574
|
+
} else {
|
|
1575
|
+
encoded = await cropped.encodeJPEG(90); // JPEG quality 90
|
|
1576
|
+
}
|
|
1577
|
+
return Buffer.from(encoded);
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/** Sentinel: the worker path could not run (worker_threads unavailable / disabled). */
|
|
1581
|
+
const WORKER_UNAVAILABLE = Symbol("worker-unavailable");
|
|
1582
|
+
|
|
1583
|
+
/** Whether to offload decode/crop/encode to a terminable worker thread. Default on. */
|
|
1584
|
+
function decodeWorkerEnabled(): boolean {
|
|
1585
|
+
const raw = process.env.PI_VISION_PROXY_DECODE_WORKER?.toLowerCase();
|
|
1586
|
+
return raw !== "0" && raw !== "false" && raw !== "no" && raw !== "off";
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
// Persistent CommonJS worker body (run via `{ eval: true }`). ImageScript is
|
|
1590
|
+
// loaded once from the path supplied in workerData, then the worker serves crop
|
|
1591
|
+
// tasks in a message loop so a pooled worker can be reused across calls without
|
|
1592
|
+
// paying decode-library init each time. Running in a worker is what makes the
|
|
1593
|
+
// timeout a *hard* limit: the main thread stays responsive and can terminate()
|
|
1594
|
+
// this thread mid-decode, which a same-thread Promise.race cannot do against
|
|
1595
|
+
// synchronous WASM.
|
|
1596
|
+
const CROP_WORKER_SRC = `
|
|
1597
|
+
const { parentPort, workerData } = require("worker_threads");
|
|
1598
|
+
const { Image } = require(workerData.imagescriptPath);
|
|
1599
|
+
parentPort.on("message", async (task) => {
|
|
1600
|
+
const { bytes, crop, mimeType, maxDim } = task;
|
|
1601
|
+
try {
|
|
1602
|
+
const img = await Image.decode(new Uint8Array(bytes));
|
|
1603
|
+
if (img.width > maxDim || img.height > maxDim) { parentPort.postMessage({ ok: false }); return; }
|
|
1604
|
+
const cropped = img.crop(crop.x, crop.y, crop.width, crop.height);
|
|
1605
|
+
const encoded = mimeType === "image/png" ? await cropped.encode(1) : await cropped.encodeJPEG(90);
|
|
1606
|
+
const u8 = encoded instanceof Uint8Array ? encoded : new Uint8Array(encoded);
|
|
1607
|
+
const out = u8.buffer.slice(u8.byteOffset, u8.byteOffset + u8.byteLength);
|
|
1608
|
+
parentPort.postMessage({ ok: true, data: out }, [out]);
|
|
1609
|
+
} catch (e) {
|
|
1610
|
+
parentPort.postMessage({ ok: false, error: String((e && e.message) || e) });
|
|
1611
|
+
}
|
|
1612
|
+
});
|
|
1613
|
+
`;
|
|
1614
|
+
|
|
1615
|
+
type NodeWorker = import("node:worker_threads").Worker;
|
|
1616
|
+
|
|
1617
|
+
/** An idle pooled worker plus the cleanup that detaches its idle-health listeners. */
|
|
1618
|
+
interface PooledWorker {
|
|
1619
|
+
worker: NodeWorker;
|
|
1620
|
+
detach: () => void;
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
/** Idle, reusable workers. Bounded by maxIdleWorkers(); unref'd so they never block process exit. */
|
|
1624
|
+
const _idleWorkers: PooledWorker[] = [];
|
|
1625
|
+
|
|
1626
|
+
/** Maximum idle workers retained between calls. 0 disables pooling (spawn-per-call). */
|
|
1627
|
+
function maxIdleWorkers(): number {
|
|
1628
|
+
const raw = process.env.PI_VISION_PROXY_DECODE_WORKER_POOL;
|
|
1629
|
+
if (raw) {
|
|
1630
|
+
const n = Number.parseInt(raw, 10);
|
|
1631
|
+
if (Number.isFinite(n) && n >= 0) return n;
|
|
1632
|
+
}
|
|
1633
|
+
return 2;
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
let _workerCtor: typeof import("node:worker_threads").Worker | null = null;
|
|
1637
|
+
let _imagescriptPath: string | null = null;
|
|
1638
|
+
let _workerInfraResolved = false;
|
|
1639
|
+
|
|
1640
|
+
/** Resolve the Worker constructor and ImageScript path once. Returns false if unavailable. */
|
|
1641
|
+
async function ensureWorkerInfra(): Promise<boolean> {
|
|
1642
|
+
if (_workerInfraResolved) return _workerCtor !== null && _imagescriptPath !== null;
|
|
1643
|
+
_workerInfraResolved = true;
|
|
1644
|
+
try {
|
|
1645
|
+
_workerCtor = (await import("node:worker_threads")).Worker;
|
|
1646
|
+
const { createRequire } = await import("node:module");
|
|
1647
|
+
_imagescriptPath = createRequire(import.meta.url).resolve("imagescript");
|
|
1648
|
+
return true;
|
|
1649
|
+
} catch {
|
|
1650
|
+
_workerCtor = null;
|
|
1651
|
+
_imagescriptPath = null;
|
|
1652
|
+
return false;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
/** Take an idle worker (detaching its health listeners) or spawn a fresh one. */
|
|
1657
|
+
function acquireWorker(): NodeWorker {
|
|
1658
|
+
const budget = maxIdleWorkers();
|
|
1659
|
+
// Honor the *current* budget before reusing anything: terminate idle workers
|
|
1660
|
+
// beyond it so a lowered PI_VISION_PROXY_DECODE_WORKER_POOL takes effect
|
|
1661
|
+
// immediately rather than waiting for the pool to drain naturally. With
|
|
1662
|
+
// budget 0 this empties the pool, making spawn-per-call truly spawn-per-call.
|
|
1663
|
+
while (_idleWorkers.length > budget) {
|
|
1664
|
+
const extra = _idleWorkers.pop()!;
|
|
1665
|
+
extra.detach();
|
|
1666
|
+
void extra.worker.terminate();
|
|
1667
|
+
}
|
|
1668
|
+
// Only reuse a pooled worker when pooling is enabled.
|
|
1669
|
+
if (budget > 0) {
|
|
1670
|
+
const pooled = _idleWorkers.pop();
|
|
1671
|
+
if (pooled) {
|
|
1672
|
+
pooled.detach();
|
|
1673
|
+
pooled.worker.ref();
|
|
1674
|
+
return pooled.worker;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
// _workerCtor / _imagescriptPath are non-null here (ensureWorkerInfra succeeded).
|
|
1678
|
+
return new _workerCtor!(CROP_WORKER_SRC, {
|
|
1679
|
+
eval: true,
|
|
1680
|
+
workerData: { imagescriptPath: _imagescriptPath },
|
|
1681
|
+
});
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
/** Return a healthy worker to the idle pool (unref'd), or terminate it if the pool is full. */
|
|
1685
|
+
function releaseWorker(worker: NodeWorker): void {
|
|
1686
|
+
if (_idleWorkers.length >= maxIdleWorkers()) {
|
|
1687
|
+
void worker.terminate();
|
|
1688
|
+
return;
|
|
1689
|
+
}
|
|
1690
|
+
// If the worker dies while idle, drop it from the pool so it is never reused.
|
|
1691
|
+
const onDeath = () => {
|
|
1692
|
+
const i = _idleWorkers.findIndex((p) => p.worker === worker);
|
|
1693
|
+
if (i >= 0) _idleWorkers.splice(i, 1);
|
|
1694
|
+
};
|
|
1695
|
+
worker.once("exit", onDeath);
|
|
1696
|
+
worker.once("error", onDeath);
|
|
1697
|
+
worker.unref();
|
|
1698
|
+
_idleWorkers.push({
|
|
1699
|
+
worker,
|
|
1700
|
+
detach: () => {
|
|
1701
|
+
worker.off("exit", onDeath);
|
|
1702
|
+
worker.off("error", onDeath);
|
|
1703
|
+
},
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
/** Run one crop task on a worker with a hard timeout. `reusable` is false on timeout/error. */
|
|
1708
|
+
function runCropTask(
|
|
1709
|
+
worker: NodeWorker,
|
|
1710
|
+
task: { bytes: ArrayBuffer; crop: ResolvedCrop; mimeType?: string; maxDim: number },
|
|
1711
|
+
timeoutMs: number,
|
|
1712
|
+
): Promise<{ result: Buffer | null; reusable: boolean }> {
|
|
1713
|
+
return new Promise((resolve) => {
|
|
1714
|
+
let settled = false;
|
|
1715
|
+
const settle = (result: Buffer | null, reusable: boolean) => {
|
|
1716
|
+
if (settled) return;
|
|
1717
|
+
settled = true;
|
|
1718
|
+
clearTimeout(timer);
|
|
1719
|
+
worker.off("message", onMessage);
|
|
1720
|
+
worker.off("error", onError);
|
|
1721
|
+
worker.off("exit", onExit);
|
|
1722
|
+
resolve({ result, reusable });
|
|
1723
|
+
};
|
|
1724
|
+
const onMessage = (msg: { ok?: boolean; data?: ArrayBuffer }) =>
|
|
1725
|
+
settle(msg && msg.ok && msg.data ? Buffer.from(msg.data) : null, true);
|
|
1726
|
+
const onError = () => settle(null, false);
|
|
1727
|
+
const onExit = () => settle(null, false);
|
|
1728
|
+
// Timeout → not reusable: the worker may be wedged in a synchronous decode.
|
|
1729
|
+
const timer = setTimeout(() => settle(null, false), timeoutMs);
|
|
1730
|
+
worker.on("message", onMessage);
|
|
1731
|
+
worker.on("error", onError);
|
|
1732
|
+
worker.on("exit", onExit);
|
|
1733
|
+
worker.postMessage(task, [task.bytes]);
|
|
1734
|
+
});
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
/**
|
|
1738
|
+
* Decode → crop → encode on a pooled, terminable worker thread with a hard
|
|
1739
|
+
* timeout. Returns the cropped bytes, null on decode/crop failure (including a
|
|
1740
|
+
* terminated timeout), or WORKER_UNAVAILABLE if worker infra is unavailable
|
|
1741
|
+
* (caller should fall back to the in-thread path).
|
|
1742
|
+
*/
|
|
1743
|
+
async function cropInWorker(
|
|
1744
|
+
imageBytes: Buffer,
|
|
1745
|
+
crop: ResolvedCrop,
|
|
1746
|
+
mimeType: string | undefined,
|
|
1747
|
+
timeoutMs: number,
|
|
1748
|
+
): Promise<Buffer | null | typeof WORKER_UNAVAILABLE> {
|
|
1749
|
+
if (!(await ensureWorkerInfra())) return WORKER_UNAVAILABLE;
|
|
1750
|
+
|
|
1751
|
+
let worker: NodeWorker;
|
|
1752
|
+
try {
|
|
1753
|
+
worker = acquireWorker();
|
|
1754
|
+
} catch {
|
|
1755
|
+
return WORKER_UNAVAILABLE;
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
// Detach a standalone, transferable copy of the bytes (Buffer pooling means
|
|
1759
|
+
// imageBytes.buffer may be shared and unsafe to transfer directly).
|
|
1760
|
+
const ab = imageBytes.buffer.slice(imageBytes.byteOffset, imageBytes.byteOffset + imageBytes.byteLength);
|
|
1761
|
+
|
|
1762
|
+
const { result, reusable } = await runCropTask(
|
|
1763
|
+
worker,
|
|
1764
|
+
{ bytes: ab, crop, mimeType, maxDim: MAX_IMAGE_DIMENSION },
|
|
1765
|
+
timeoutMs,
|
|
1766
|
+
);
|
|
1767
|
+
if (reusable) releaseWorker(worker);
|
|
1768
|
+
else void worker.terminate();
|
|
1769
|
+
return result;
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
/**
|
|
1773
|
+
* Terminate all idle pooled workers. Exposed for test teardown; safe to call
|
|
1774
|
+
* anytime (a fresh worker is spawned on the next crop).
|
|
1775
|
+
*/
|
|
1776
|
+
export async function shutdownCropWorkers(): Promise<void> {
|
|
1777
|
+
const pending = _idleWorkers.splice(0, _idleWorkers.length);
|
|
1778
|
+
await Promise.all(pending.map((p) => {
|
|
1779
|
+
p.detach();
|
|
1780
|
+
return p.worker.terminate();
|
|
1781
|
+
}));
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1345
1784
|
/**
|
|
1346
1785
|
* Crop an image buffer to the given pixel rectangle using ImageScript.
|
|
1347
1786
|
* Accepts raw image bytes (JPEG/PNG) and returns cropped bytes in the same format.
|
|
1348
1787
|
* Returns null if cropping fails.
|
|
1788
|
+
*
|
|
1789
|
+
* The decode/crop/encode runs in a terminable worker thread so a maliciously
|
|
1790
|
+
* crafted image that makes the synchronous WASM decoder spin can be killed at the
|
|
1791
|
+
* timeout instead of freezing the session. If worker_threads is unavailable (or
|
|
1792
|
+
* disabled via PI_VISION_PROXY_DECODE_WORKER=0) it falls back to the in-thread
|
|
1793
|
+
* path, which is still guarded by the dimension pre-check and decode timeout.
|
|
1349
1794
|
*/
|
|
1350
1795
|
export async function cropImage(
|
|
1351
1796
|
imageBytes: Buffer,
|
|
@@ -1358,20 +1803,12 @@ export async function cropImage(
|
|
|
1358
1803
|
if (dims && (dims.width > MAX_IMAGE_DIMENSION || dims.height > MAX_IMAGE_DIMENSION)) {
|
|
1359
1804
|
return null;
|
|
1360
1805
|
}
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
}
|
|
1366
|
-
const cropped = img.crop(crop.x, crop.y, crop.width, crop.height);
|
|
1367
|
-
// Encode back to the same format
|
|
1368
|
-
let encoded: Uint8Array;
|
|
1369
|
-
if (mimeType === "image/png") {
|
|
1370
|
-
encoded = await cropped.encode(1); // PNG with compression level 1 (fast)
|
|
1371
|
-
} else {
|
|
1372
|
-
encoded = await cropped.encodeJPEG(90); // JPEG quality 90
|
|
1806
|
+
if (decodeWorkerEnabled()) {
|
|
1807
|
+
const viaWorker = await cropInWorker(imageBytes, crop, mimeType, decodeTimeoutMs());
|
|
1808
|
+
if (viaWorker !== WORKER_UNAVAILABLE) return viaWorker;
|
|
1809
|
+
// else: worker infra unavailable — fall through to in-thread crop
|
|
1373
1810
|
}
|
|
1374
|
-
return
|
|
1811
|
+
return await cropInThread(imageBytes, crop, mimeType);
|
|
1375
1812
|
} catch {
|
|
1376
1813
|
return null;
|
|
1377
1814
|
}
|