pi-multimodal-proxy 1.6.0-beta.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +242 -219
- package/extensions/__tests__/integration.test.ts +0 -2
- package/extensions/__tests__/internal.test.ts +515 -23
- package/extensions/internal.ts +657 -78
- package/extensions/vision-proxy.ts +2493 -2076
- package/package.json +2 -2
|
@@ -8,15 +8,22 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { strict as assert } from "node:assert";
|
|
11
|
-
import { describe, it } from "node:test";
|
|
12
|
-
import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
|
11
|
+
import { after, describe, it } from "node:test";
|
|
12
|
+
import { lstat, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
|
13
13
|
import os from "node:os";
|
|
14
|
-
import { join } from "node:path";
|
|
14
|
+
import { join, parse } from "node:path";
|
|
15
15
|
import {
|
|
16
16
|
buildConversationContext,
|
|
17
17
|
buildDescriptionFence,
|
|
18
18
|
buildAnalysisFence,
|
|
19
|
+
buildVideoDescriptionFence,
|
|
20
|
+
buildVideoEmptyResponseError,
|
|
21
|
+
buildVideoProxySection,
|
|
19
22
|
clampPixels,
|
|
23
|
+
extractXaiResponsesText,
|
|
24
|
+
formatXaiSttTranscript,
|
|
25
|
+
isTranscriptionRequest,
|
|
26
|
+
isXaiProvider,
|
|
20
27
|
CUSTOM_TYPE_CONFIG,
|
|
21
28
|
CUSTOM_TYPE_CONSENT,
|
|
22
29
|
CUSTOM_TYPE_DESCRIPTION,
|
|
@@ -25,6 +32,8 @@ import {
|
|
|
25
32
|
envFlags,
|
|
26
33
|
escapeAttr,
|
|
27
34
|
extractCandidateImagePaths,
|
|
35
|
+
extractCandidateVideoPaths,
|
|
36
|
+
extractCandidateAudioPaths,
|
|
28
37
|
extractDimensions,
|
|
29
38
|
fenceUntrusted,
|
|
30
39
|
findDescriptions,
|
|
@@ -33,6 +42,7 @@ import {
|
|
|
33
42
|
hasConsent,
|
|
34
43
|
hashImageData,
|
|
35
44
|
IMAGE_PATH_PLACEHOLDER,
|
|
45
|
+
VIDEO_PATH_PLACEHOLDER,
|
|
36
46
|
isPathAllowed,
|
|
37
47
|
isValidNamedRegion,
|
|
38
48
|
LRUCache,
|
|
@@ -49,17 +59,27 @@ import {
|
|
|
49
59
|
hammingDistance,
|
|
50
60
|
computePHash,
|
|
51
61
|
cropImage,
|
|
62
|
+
shutdownCropWorkers,
|
|
52
63
|
piAiImageToBuffer,
|
|
53
64
|
bufferToPiAiImage,
|
|
54
65
|
shouldStripImages,
|
|
55
66
|
splitSubcommand,
|
|
56
67
|
stripImagePaths,
|
|
68
|
+
stripMediaPaths,
|
|
57
69
|
toPiAiImage,
|
|
58
70
|
type VisionConfig,
|
|
59
71
|
writePersistentFile,
|
|
60
72
|
sanitizeForLog,
|
|
61
73
|
storeImageMeta,
|
|
62
|
-
|
|
74
|
+
createImageMetaStore,
|
|
75
|
+
storeImageData,
|
|
76
|
+
getImageData,
|
|
77
|
+
createImageDataStore,
|
|
78
|
+
parseRecallRef,
|
|
79
|
+
spinnerFrame,
|
|
80
|
+
formatProgressStatus,
|
|
81
|
+
SPINNER_FRAMES,
|
|
82
|
+
RECALL_HINT,
|
|
63
83
|
} from "../internal.ts";
|
|
64
84
|
|
|
65
85
|
// SessionEntry minimal shape — typed loose because peer dep types are not loaded in test
|
|
@@ -89,6 +109,17 @@ describe("parseModelString", () => {
|
|
|
89
109
|
});
|
|
90
110
|
});
|
|
91
111
|
|
|
112
|
+
it("normalizes legacy x-ai provider id to Pi's xai provider", () => {
|
|
113
|
+
assert.deepEqual(parseModelString("x-ai/grok-4.3"), {
|
|
114
|
+
provider: "xai",
|
|
115
|
+
modelId: "grok-4.3",
|
|
116
|
+
});
|
|
117
|
+
assert.deepEqual(parseModelString("xai/grok-4.3"), {
|
|
118
|
+
provider: "xai",
|
|
119
|
+
modelId: "grok-4.3",
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
92
123
|
it("rejects malformed strings", () => {
|
|
93
124
|
assert.equal(parseModelString(""), null);
|
|
94
125
|
assert.equal(parseModelString("/foo"), null);
|
|
@@ -115,6 +146,12 @@ describe("sanitize", () => {
|
|
|
115
146
|
assert.equal(out.includeContext, DEFAULT_CONFIG.includeContext);
|
|
116
147
|
});
|
|
117
148
|
|
|
149
|
+
it("normalizes legacy x-ai provider id in config", () => {
|
|
150
|
+
const result = sanitize({ ...DEFAULT_CONFIG, provider: "x-ai", videoProvider: "x-ai" });
|
|
151
|
+
assert.equal(result.provider, "xai");
|
|
152
|
+
assert.equal(result.videoProvider, "xai");
|
|
153
|
+
});
|
|
154
|
+
|
|
118
155
|
it("preserves valid values", () => {
|
|
119
156
|
const cfg: VisionConfig = {
|
|
120
157
|
mode: "always",
|
|
@@ -163,6 +200,12 @@ describe("readEnvOverrides", () => {
|
|
|
163
200
|
assert.equal(out.modelId, "gpt-4o");
|
|
164
201
|
});
|
|
165
202
|
|
|
203
|
+
it("normalizes legacy x-ai video model env override", () => {
|
|
204
|
+
const out = readEnvOverrides({ PI_VISION_PROXY_VIDEO_MODEL: "x-ai/grok-4.3" });
|
|
205
|
+
assert.equal(out.videoProvider, "xai");
|
|
206
|
+
assert.equal(out.videoModelId, "grok-4.3");
|
|
207
|
+
});
|
|
208
|
+
|
|
166
209
|
it("ignores malformed model string", () => {
|
|
167
210
|
assert.deepEqual(readEnvOverrides({ PI_VISION_PROXY_MODEL: "noslash" }), {});
|
|
168
211
|
});
|
|
@@ -251,6 +294,152 @@ describe("pluralImages", () => {
|
|
|
251
294
|
});
|
|
252
295
|
});
|
|
253
296
|
|
|
297
|
+
describe("parseRecallRef", () => {
|
|
298
|
+
const hash = "a".repeat(32);
|
|
299
|
+
|
|
300
|
+
it("accepts a bare 32-hex hash", () => {
|
|
301
|
+
assert.equal(parseRecallRef(hash), hash);
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
it("accepts a sha256:-prefixed hash", () => {
|
|
305
|
+
assert.equal(parseRecallRef(`sha256:${hash}`), hash);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
it("strips a #crop suffix and recalls the base image", () => {
|
|
309
|
+
assert.equal(parseRecallRef(`${hash}#crop:1840,120,840,360`), hash);
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
it("normalizes uppercase to lowercase", () => {
|
|
313
|
+
assert.equal(parseRecallRef("A".repeat(32)), hash);
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
it("rejects file paths and non-hash refs", () => {
|
|
317
|
+
assert.equal(parseRecallRef("/tmp/shot.png"), null);
|
|
318
|
+
assert.equal(parseRecallRef("./a.png"), null);
|
|
319
|
+
assert.equal(parseRecallRef("screenshot.png"), null);
|
|
320
|
+
// Wrong length / non-hex
|
|
321
|
+
assert.equal(parseRecallRef("a".repeat(31)), null);
|
|
322
|
+
assert.equal(parseRecallRef("a".repeat(33)), null);
|
|
323
|
+
assert.equal(parseRecallRef("z".repeat(32)), null);
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
describe("session image recall store", () => {
|
|
328
|
+
it("round-trips retained image bytes by hash", () => {
|
|
329
|
+
const store = createImageDataStore();
|
|
330
|
+
storeImageData(store, "hash1", "AAAA", "image/png");
|
|
331
|
+
const got = getImageData(store, "hash1");
|
|
332
|
+
assert.deepEqual(got, { data: "AAAA", mimeType: "image/png" });
|
|
333
|
+
assert.equal(getImageData(store, "missing"), undefined);
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
it("ignores empty hash or data", () => {
|
|
337
|
+
const store = createImageDataStore();
|
|
338
|
+
storeImageData(store, "", "AAAA", "image/png");
|
|
339
|
+
storeImageData(store, "hash", "", "image/png");
|
|
340
|
+
assert.equal(store.map.size, 0);
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
it("does not duplicate on re-store of the same hash", () => {
|
|
344
|
+
const store = createImageDataStore();
|
|
345
|
+
storeImageData(store, "hash1", "AAAA", "image/png");
|
|
346
|
+
storeImageData(store, "hash1", "AAAA", "image/png");
|
|
347
|
+
assert.equal(store.map.size, 1);
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
it("keeps stores isolated — one session's bytes do not leak into another (issue #12)", () => {
|
|
351
|
+
const sessionA = createImageDataStore();
|
|
352
|
+
const sessionB = createImageDataStore();
|
|
353
|
+
storeImageData(sessionA, "hash1", "AAAA", "image/png");
|
|
354
|
+
assert.deepEqual(getImageData(sessionA, "hash1"), { data: "AAAA", mimeType: "image/png" });
|
|
355
|
+
assert.equal(getImageData(sessionB, "hash1"), undefined);
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it("evicts least-recently-used entries past the byte budget", () => {
|
|
359
|
+
const store = createImageDataStore();
|
|
360
|
+
const prev = process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
361
|
+
// Budget of 10 decoded bytes; each 8-char base64 entry decodes to 6 bytes.
|
|
362
|
+
process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = "10";
|
|
363
|
+
try {
|
|
364
|
+
storeImageData(store, "h1", "AAAAAAAA", "image/png"); // 6 decoded bytes, total 6
|
|
365
|
+
storeImageData(store, "h2", "BBBBBBBB", "image/png"); // 6 decoded bytes, total 12 > 10 → evict h1
|
|
366
|
+
assert.equal(getImageData(store, "h1"), undefined);
|
|
367
|
+
assert.deepEqual(getImageData(store, "h2"), { data: "BBBBBBBB", mimeType: "image/png" });
|
|
368
|
+
} finally {
|
|
369
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
370
|
+
else process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = prev;
|
|
371
|
+
}
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it("keeps a single oversized image rather than evicting everything", () => {
|
|
375
|
+
const store = createImageDataStore();
|
|
376
|
+
const prev = process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
377
|
+
process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = "4";
|
|
378
|
+
try {
|
|
379
|
+
storeImageData(store, "big", "AAAAAAAAAAAA", "image/png"); // 9 decoded bytes > 4 budget
|
|
380
|
+
assert.deepEqual(getImageData(store, "big"), { data: "AAAAAAAAAAAA", mimeType: "image/png" });
|
|
381
|
+
assert.equal(store.map.size, 1);
|
|
382
|
+
} finally {
|
|
383
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
384
|
+
else process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = prev;
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
it("bumps recency on access so the touched entry survives eviction", () => {
|
|
389
|
+
const store = createImageDataStore();
|
|
390
|
+
const prev = process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
391
|
+
// Budget of 14 decoded bytes; each 8-char base64 entry decodes to 6 bytes.
|
|
392
|
+
process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = "14";
|
|
393
|
+
try {
|
|
394
|
+
storeImageData(store, "h1", "AAAAAAAA", "image/png"); // 6 decoded bytes, total 6
|
|
395
|
+
storeImageData(store, "h2", "BBBBBBBB", "image/png"); // 6 decoded bytes, total 12
|
|
396
|
+
getImageData(store, "h1"); // bump h1 to most-recent
|
|
397
|
+
storeImageData(store, "h3", "CCCCCCCC", "image/png"); // 6 decoded bytes, total 18 > 14 → evict LRU (h2)
|
|
398
|
+
assert.deepEqual(getImageData(store, "h1"), { data: "AAAAAAAA", mimeType: "image/png" });
|
|
399
|
+
assert.equal(getImageData(store, "h2"), undefined);
|
|
400
|
+
assert.deepEqual(getImageData(store, "h3"), { data: "CCCCCCCC", mimeType: "image/png" });
|
|
401
|
+
} finally {
|
|
402
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES;
|
|
403
|
+
else process.env.PI_VISION_PROXY_IMAGE_RECALL_BYTES = prev;
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
describe("spinnerFrame", () => {
|
|
409
|
+
it("returns a frame from the set and wraps around", () => {
|
|
410
|
+
assert.equal(spinnerFrame(0), SPINNER_FRAMES[0]);
|
|
411
|
+
assert.equal(spinnerFrame(SPINNER_FRAMES.length), SPINNER_FRAMES[0]);
|
|
412
|
+
assert.equal(spinnerFrame(SPINNER_FRAMES.length + 1), SPINNER_FRAMES[1]);
|
|
413
|
+
assert.ok(SPINNER_FRAMES.includes(spinnerFrame(7)));
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
it("handles negative ticks without throwing", () => {
|
|
417
|
+
assert.ok(SPINNER_FRAMES.includes(spinnerFrame(-1)));
|
|
418
|
+
assert.ok(SPINNER_FRAMES.includes(spinnerFrame(-13)));
|
|
419
|
+
});
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
describe("formatProgressStatus", () => {
|
|
423
|
+
it("includes frame, label, and elapsed seconds", () => {
|
|
424
|
+
assert.equal(
|
|
425
|
+
formatProgressStatus("Analyzing image 2/4…", "⠙", 3),
|
|
426
|
+
"multimodal-proxy ⠙ Analyzing image 2/4… (3s)",
|
|
427
|
+
);
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
it("floors and clamps elapsed seconds", () => {
|
|
431
|
+
assert.match(formatProgressStatus("x", "⠋", 4.9), /\(4s\)$/);
|
|
432
|
+
assert.match(formatProgressStatus("x", "⠋", -2), /\(0s\)$/);
|
|
433
|
+
});
|
|
434
|
+
});
|
|
435
|
+
|
|
436
|
+
describe("RECALL_HINT", () => {
|
|
437
|
+
it("mentions analyze_image and the image id", () => {
|
|
438
|
+
assert.match(RECALL_HINT, /analyze_image/);
|
|
439
|
+
assert.match(RECALL_HINT, /image id/i);
|
|
440
|
+
});
|
|
441
|
+
});
|
|
442
|
+
|
|
254
443
|
describe("splitSubcommand", () => {
|
|
255
444
|
it("splits sub and value with arbitrary whitespace", () => {
|
|
256
445
|
assert.deepEqual(splitSubcommand("model anthropic/claude"), { sub: "model", value: "anthropic/claude" });
|
|
@@ -469,6 +658,46 @@ describe("extractCandidateImagePaths", () => {
|
|
|
469
658
|
});
|
|
470
659
|
});
|
|
471
660
|
|
|
661
|
+
describe("extractCandidateMediaPaths", () => {
|
|
662
|
+
it("detects quoted Windows video paths with spaces", () => {
|
|
663
|
+
const input = `video is not working: "D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4" transcribe this`;
|
|
664
|
+
assert.deepEqual(extractCandidateVideoPaths(input), [
|
|
665
|
+
"D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4",
|
|
666
|
+
]);
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
it("detects unquoted video paths without spaces", () => {
|
|
670
|
+
assert.deepEqual(extractCandidateVideoPaths("see D:\\Downloads\\clip.mp4 now"), ["D:\\Downloads\\clip.mp4"]);
|
|
671
|
+
assert.deepEqual(extractCandidateVideoPaths("see ./clip.mkv now"), ["./clip.mkv"]);
|
|
672
|
+
});
|
|
673
|
+
|
|
674
|
+
it("detects unquoted Windows video paths with spaces", () => {
|
|
675
|
+
const input = "Transcribe this video with timestamps D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4";
|
|
676
|
+
assert.deepEqual(extractCandidateVideoPaths(input), [
|
|
677
|
+
"D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4",
|
|
678
|
+
]);
|
|
679
|
+
});
|
|
680
|
+
|
|
681
|
+
it("detects quoted relative paths and audio paths with spaces", () => {
|
|
682
|
+
assert.deepEqual(extractCandidateVideoPaths(`see "./my video.mp4" now`), ["./my video.mp4"]);
|
|
683
|
+
assert.deepEqual(extractCandidateAudioPaths(`listen "C:\\Users\\Me\\Audio File.m4a" please`), [
|
|
684
|
+
"C:\\Users\\Me\\Audio File.m4a",
|
|
685
|
+
]);
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
it("does not treat unquoted paths with spaces as a single path", () => {
|
|
689
|
+
assert.deepEqual(extractCandidateVideoPaths("see ./my video.mp4 now"), []);
|
|
690
|
+
});
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
describe("stripMediaPaths", () => {
|
|
694
|
+
it("replaces media paths with placeholder", () => {
|
|
695
|
+
const mediaPath = "D:\\Downloads\\Rethinking Agents - Harness is All you Need_.mp4";
|
|
696
|
+
const result = stripMediaPaths(`transcribe "${mediaPath}" please`, [mediaPath]);
|
|
697
|
+
assert.equal(result, `transcribe "${VIDEO_PATH_PLACEHOLDER}" please`);
|
|
698
|
+
});
|
|
699
|
+
});
|
|
700
|
+
|
|
472
701
|
describe("stripImagePaths", () => {
|
|
473
702
|
it("replaces a single path with placeholder", () => {
|
|
474
703
|
const result = stripImagePaths("see /tmp/pi-clipboard-abc.png here", ["/tmp/pi-clipboard-abc.png"]);
|
|
@@ -517,29 +746,79 @@ describe("isPathAllowed", () => {
|
|
|
517
746
|
}
|
|
518
747
|
});
|
|
519
748
|
|
|
520
|
-
it("
|
|
521
|
-
|
|
749
|
+
it("allows non-existent files whose parent directory is in the allow-list", async () => {
|
|
750
|
+
// Non-existent files in tmpdir pass the check so callers can return "unreadable"
|
|
751
|
+
// instead of the misleading "denied" / "path outside allowed directories" message.
|
|
752
|
+
assert.equal(await isPathAllowed(join(os.tmpdir(), "does-not-exist-xyz.png")), true);
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
it("denies non-existent files outside the allow-list", async () => {
|
|
756
|
+
assert.equal(await isPathAllowed("/etc/does-not-exist-vp-xyz.png"), false);
|
|
757
|
+
});
|
|
758
|
+
|
|
759
|
+
it("allows files inside /tmp (system-wide Unix temp dir)", async () => {
|
|
760
|
+
if (os.platform() === "win32") return;
|
|
761
|
+
const file = join("/tmp", `vp-test-direct-${Date.now()}.png`);
|
|
762
|
+
await writeFile(file, TINY_PNG);
|
|
763
|
+
try {
|
|
764
|
+
assert.equal(await isPathAllowed(file), true);
|
|
765
|
+
} finally {
|
|
766
|
+
try { await rm(file); } catch { /* ignore */ }
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
it("allows non-existent files inside /tmp", async () => {
|
|
771
|
+
if (os.platform() === "win32") return;
|
|
772
|
+
assert.equal(await isPathAllowed("/tmp/does-not-exist-vp-xyz.png"), true);
|
|
773
|
+
});
|
|
774
|
+
|
|
775
|
+
it("allows local Windows drive paths by default", async () => {
|
|
776
|
+
const root = parse(process.cwd()).root;
|
|
777
|
+
if (!/^[a-z]:[\\/]/i.test(root)) return;
|
|
778
|
+
const prevDrives = process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
779
|
+
try {
|
|
780
|
+
delete process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
781
|
+
assert.equal(await isPathAllowed(process.cwd()), true);
|
|
782
|
+
} finally {
|
|
783
|
+
if (prevDrives === undefined) delete process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
784
|
+
else process.env.PI_VISION_PROXY_ALLOW_DRIVES = prevDrives;
|
|
785
|
+
}
|
|
786
|
+
});
|
|
787
|
+
|
|
788
|
+
it("can disable local Windows drive path access with PI_VISION_PROXY_ALLOW_DRIVES=0", async () => {
|
|
789
|
+
const root = parse(process.cwd()).root;
|
|
790
|
+
if (!/^[a-z]:[\\/]/i.test(root)) return;
|
|
791
|
+
const prevDrives = process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
792
|
+
try {
|
|
793
|
+
process.env.PI_VISION_PROXY_ALLOW_DRIVES = "0";
|
|
794
|
+
// cwd is still allowed by the cwd rule, so assert using the user home when it is outside cwd.
|
|
795
|
+
const home = os.homedir();
|
|
796
|
+
if (!home.toLowerCase().startsWith(process.cwd().toLowerCase())) {
|
|
797
|
+
assert.equal(await isPathAllowed(home), false);
|
|
798
|
+
}
|
|
799
|
+
} finally {
|
|
800
|
+
if (prevDrives === undefined) delete process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
801
|
+
else process.env.PI_VISION_PROXY_ALLOW_DRIVES = prevDrives;
|
|
802
|
+
}
|
|
522
803
|
});
|
|
523
804
|
|
|
524
|
-
it("
|
|
525
|
-
// Use tmpdir as a stand-in we can write to; flip env to simulate the gate.
|
|
526
|
-
// We resolve a path that is neither under cwd nor tmp by asserting the env behaviour
|
|
527
|
-
// indirectly: with the flag set, an existing tmp file is still allowed (tmp wins);
|
|
528
|
-
// without it, that's also true. So we test the env path explicitly with realpath of
|
|
529
|
-
// homedir itself, which is a real, resolvable directory outside tmp/cwd.
|
|
805
|
+
it("allows homedir files when PI_VISION_PROXY_ALLOW_HOME=1", async () => {
|
|
530
806
|
const home = os.homedir();
|
|
531
|
-
const
|
|
807
|
+
const prevHome = process.env.PI_VISION_PROXY_ALLOW_HOME;
|
|
808
|
+
const prevDrives = process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
532
809
|
try {
|
|
810
|
+
process.env.PI_VISION_PROXY_ALLOW_DRIVES = "0";
|
|
533
811
|
delete process.env.PI_VISION_PROXY_ALLOW_HOME;
|
|
534
|
-
// homedir() may equal cwd in odd setups; skip the assertion in that case.
|
|
535
812
|
if (!home.toLowerCase().startsWith(process.cwd().toLowerCase())) {
|
|
536
813
|
assert.equal(await isPathAllowed(home), false);
|
|
537
814
|
}
|
|
538
815
|
process.env.PI_VISION_PROXY_ALLOW_HOME = "1";
|
|
539
816
|
assert.equal(await isPathAllowed(home), true);
|
|
540
817
|
} finally {
|
|
541
|
-
if (
|
|
542
|
-
else process.env.PI_VISION_PROXY_ALLOW_HOME =
|
|
818
|
+
if (prevHome === undefined) delete process.env.PI_VISION_PROXY_ALLOW_HOME;
|
|
819
|
+
else process.env.PI_VISION_PROXY_ALLOW_HOME = prevHome;
|
|
820
|
+
if (prevDrives === undefined) delete process.env.PI_VISION_PROXY_ALLOW_DRIVES;
|
|
821
|
+
else process.env.PI_VISION_PROXY_ALLOW_DRIVES = prevDrives;
|
|
543
822
|
}
|
|
544
823
|
});
|
|
545
824
|
});
|
|
@@ -567,13 +846,20 @@ describe("readImageFileWithReason", () => {
|
|
|
567
846
|
});
|
|
568
847
|
|
|
569
848
|
it("returns reason=denied for path outside allow-list", async () => {
|
|
570
|
-
// /etc/passwd.png does not exist but extension is image-like.
|
|
571
|
-
// realpath fails → denied. Either reason is acceptable in that order; assert non-null reason.
|
|
572
849
|
const r = await readImageFileWithReason("/etc/never-exists-vp.png");
|
|
573
850
|
assert.equal(r.image, null);
|
|
574
851
|
assert.equal(r.reason, "denied");
|
|
575
852
|
});
|
|
576
853
|
|
|
854
|
+
it("returns reason=unreadable for non-existent file inside /tmp", async () => {
|
|
855
|
+
if (os.platform() === "win32") return;
|
|
856
|
+
// Previously returned "denied" (misleading); now returns "unreadable" so the user
|
|
857
|
+
// knows the file is simply missing, not that /tmp itself is forbidden.
|
|
858
|
+
const r = await readImageFileWithReason("/tmp/does-not-exist-vp-xyz.png");
|
|
859
|
+
assert.equal(r.image, null);
|
|
860
|
+
assert.equal(r.reason, "unreadable");
|
|
861
|
+
});
|
|
862
|
+
|
|
577
863
|
it("returns reason=empty for zero-byte image", async () => {
|
|
578
864
|
const dir = await mkdtemp(join(os.tmpdir(), "vp-test-"));
|
|
579
865
|
const file = join(dir, "empty.png");
|
|
@@ -622,6 +908,34 @@ describe("readImageFileWithReason", () => {
|
|
|
622
908
|
await rm(dir, { recursive: true, force: true });
|
|
623
909
|
}
|
|
624
910
|
});
|
|
911
|
+
|
|
912
|
+
it("denies symlink to existing file outside allow-list (TOCTOU post-read check)", async () => {
|
|
913
|
+
// Simulate the TOCTOU race: the symlink target exists and is readable, so
|
|
914
|
+
// readFile() succeeds, but the post-read realpath re-verification must catch that
|
|
915
|
+
// the resolved path is outside the allow-list and return "denied".
|
|
916
|
+
if (os.platform() === "win32") return;
|
|
917
|
+
// Find a readable file outside the allow-list to use as a target.
|
|
918
|
+
// /etc/hostname is present on most Unix systems; use it if it exists.
|
|
919
|
+
const target = "/etc/hostname";
|
|
920
|
+
let targetExists = false;
|
|
921
|
+
try { await lstat(target); targetExists = true; } catch { /* skip if absent */ }
|
|
922
|
+
if (!targetExists) return;
|
|
923
|
+
|
|
924
|
+
const dir = await mkdtemp(join(os.tmpdir(), "vp-test-"));
|
|
925
|
+
const link = join(dir, "link.png");
|
|
926
|
+
try {
|
|
927
|
+
try {
|
|
928
|
+
await symlink(target, link);
|
|
929
|
+
} catch {
|
|
930
|
+
return; // no symlink support → skip
|
|
931
|
+
}
|
|
932
|
+
const r = await readImageFileWithReason(link);
|
|
933
|
+
assert.equal(r.image, null);
|
|
934
|
+
assert.equal(r.reason, "denied");
|
|
935
|
+
} finally {
|
|
936
|
+
await rm(dir, { recursive: true, force: true });
|
|
937
|
+
}
|
|
938
|
+
});
|
|
625
939
|
});
|
|
626
940
|
|
|
627
941
|
describe("readPersistentFile / writePersistentFile", () => {
|
|
@@ -954,6 +1268,82 @@ describe("extractDimensions", () => {
|
|
|
954
1268
|
});
|
|
955
1269
|
});
|
|
956
1270
|
|
|
1271
|
+
describe("buildVideoDescriptionFence", () => {
|
|
1272
|
+
it("builds video fence with file, hash, and mime attributes", () => {
|
|
1273
|
+
const fence = buildVideoDescriptionFence("abc123", "clip.mp4", "video/mp4", "Hello world");
|
|
1274
|
+
assert.ok(fence.includes('file="clip.mp4"'));
|
|
1275
|
+
assert.ok(fence.includes('hash="abc123"'));
|
|
1276
|
+
assert.ok(fence.includes('mime="video/mp4"'));
|
|
1277
|
+
assert.ok(fence.includes("Hello world"));
|
|
1278
|
+
});
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
describe("buildVideoEmptyResponseError", () => {
|
|
1282
|
+
it("returns actionable guidance for providers that accept video but return no text", () => {
|
|
1283
|
+
const message = buildVideoEmptyResponseError("xai", "grok-4.3");
|
|
1284
|
+
assert.ok(message.includes("empty response from xai/grok-4.3"));
|
|
1285
|
+
assert.ok(message.includes("provider accepted the request but returned no text"));
|
|
1286
|
+
assert.ok(message.includes("native xAI STT or Files/Responses path"));
|
|
1287
|
+
assert.ok(message.includes("shorter clip"));
|
|
1288
|
+
assert.ok(message.includes("smaller/transcoded video"));
|
|
1289
|
+
assert.ok(message.includes("Gemini"));
|
|
1290
|
+
});
|
|
1291
|
+
});
|
|
1292
|
+
|
|
1293
|
+
describe("xAI native media helpers", () => {
|
|
1294
|
+
it("detects transcription requests", () => {
|
|
1295
|
+
assert.equal(isTranscriptionRequest("Transcribe this video with timestamps"), true);
|
|
1296
|
+
assert.equal(isTranscriptionRequest("make SRT captions"), true);
|
|
1297
|
+
assert.equal(isTranscriptionRequest("summarize visual scenes"), false);
|
|
1298
|
+
});
|
|
1299
|
+
|
|
1300
|
+
it("detects xAI provider aliases", () => {
|
|
1301
|
+
assert.equal(isXaiProvider("xai"), true);
|
|
1302
|
+
assert.equal(isXaiProvider("x-ai"), true);
|
|
1303
|
+
assert.equal(isXaiProvider("google"), false);
|
|
1304
|
+
});
|
|
1305
|
+
|
|
1306
|
+
it("formats xAI STT words into timestamped transcript", () => {
|
|
1307
|
+
const formatted = formatXaiSttTranscript({
|
|
1308
|
+
text: "Hello world.",
|
|
1309
|
+
language: "English",
|
|
1310
|
+
duration: 1.5,
|
|
1311
|
+
words: [
|
|
1312
|
+
{ text: "Hello", start: 0.1, end: 0.5 },
|
|
1313
|
+
{ text: "world.", start: 0.6, end: 1.2 },
|
|
1314
|
+
],
|
|
1315
|
+
}, "clip.mp4");
|
|
1316
|
+
assert.ok(formatted.includes("xAI Speech-to-Text transcription for clip.mp4"));
|
|
1317
|
+
assert.ok(formatted.includes("Detected language: English"));
|
|
1318
|
+
assert.ok(formatted.includes("[00:00"));
|
|
1319
|
+
assert.ok(formatted.includes("Hello world."));
|
|
1320
|
+
});
|
|
1321
|
+
|
|
1322
|
+
it("extracts output text from xAI Responses objects", () => {
|
|
1323
|
+
assert.equal(extractXaiResponsesText({ output_text: "direct" }), "direct");
|
|
1324
|
+
assert.equal(extractXaiResponsesText({ output: [{ type: "message", content: [{ type: "output_text", text: "nested" }] }] }), "nested");
|
|
1325
|
+
});
|
|
1326
|
+
});
|
|
1327
|
+
|
|
1328
|
+
describe("buildVideoProxySection", () => {
|
|
1329
|
+
it("instructs downstream agents to use injected video analysis instead of local transcription tools", () => {
|
|
1330
|
+
const section = buildVideoProxySection(
|
|
1331
|
+
1,
|
|
1332
|
+
"x-ai",
|
|
1333
|
+
"grok-4.3",
|
|
1334
|
+
buildVideoDescriptionFence("abc123", "clip.mp4", "video/mp4", "[00:00] hello"),
|
|
1335
|
+
);
|
|
1336
|
+
assert.ok(section.includes("already analyzed the media"));
|
|
1337
|
+
assert.ok(section.includes("answer from this injected context"));
|
|
1338
|
+
assert.ok(section.includes("Do not run local media-processing or transcription tools"));
|
|
1339
|
+
assert.ok(section.includes("ffmpeg"));
|
|
1340
|
+
assert.ok(section.includes("Whisper"));
|
|
1341
|
+
assert.ok(section.includes("Python"));
|
|
1342
|
+
assert.ok(section.includes("Only use external/local tools"));
|
|
1343
|
+
assert.ok(section.includes("<vision_proxy_video_description"));
|
|
1344
|
+
});
|
|
1345
|
+
});
|
|
1346
|
+
|
|
957
1347
|
describe("buildDescriptionFence", () => {
|
|
958
1348
|
it("builds fence with metadata attributes", () => {
|
|
959
1349
|
const fence = buildDescriptionFence("abc123", "A screenshot", { width: 1920, height: 1080, filename: "screen.png" });
|
|
@@ -1143,6 +1533,10 @@ async function create10x10Png(): Promise<Buffer> {
|
|
|
1143
1533
|
}
|
|
1144
1534
|
|
|
1145
1535
|
describe("cropImage (ImageScript)", () => {
|
|
1536
|
+
after(async () => {
|
|
1537
|
+
await shutdownCropWorkers();
|
|
1538
|
+
});
|
|
1539
|
+
|
|
1146
1540
|
it("crops a 10×10 PNG to a 5×5 region", async () => {
|
|
1147
1541
|
const png = await create10x10Png();
|
|
1148
1542
|
const crop = { x: 2, y: 3, width: 5, height: 5 };
|
|
@@ -1177,6 +1571,92 @@ describe("cropImage (ImageScript)", () => {
|
|
|
1177
1571
|
assert.equal(result[0], 0xff);
|
|
1178
1572
|
assert.equal(result[1], 0xd8);
|
|
1179
1573
|
});
|
|
1574
|
+
|
|
1575
|
+
it("succeeds within a generous decode timeout (env override is honoured)", async () => {
|
|
1576
|
+
const prev = process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS;
|
|
1577
|
+
process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS = "10000";
|
|
1578
|
+
try {
|
|
1579
|
+
const png = await create10x10Png();
|
|
1580
|
+
const result = await cropImage(png, { x: 0, y: 0, width: 10, height: 10 }, "image/png");
|
|
1581
|
+
assert.ok(result, "crop should succeed with a generous timeout");
|
|
1582
|
+
} finally {
|
|
1583
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS;
|
|
1584
|
+
else process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS = prev;
|
|
1585
|
+
}
|
|
1586
|
+
});
|
|
1587
|
+
|
|
1588
|
+
it("returns null (no throw) for undecodable garbage bytes", async () => {
|
|
1589
|
+
const garbage = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05]);
|
|
1590
|
+
const result = await cropImage(garbage, { x: 0, y: 0, width: 5, height: 5 }, "image/png");
|
|
1591
|
+
assert.equal(result, null);
|
|
1592
|
+
});
|
|
1593
|
+
|
|
1594
|
+
it("falls back to the in-thread path when the worker is disabled", async () => {
|
|
1595
|
+
const prev = process.env.PI_VISION_PROXY_DECODE_WORKER;
|
|
1596
|
+
process.env.PI_VISION_PROXY_DECODE_WORKER = "0";
|
|
1597
|
+
try {
|
|
1598
|
+
const png = await create10x10Png();
|
|
1599
|
+
const result = await cropImage(png, { x: 2, y: 3, width: 5, height: 5 }, "image/png");
|
|
1600
|
+
assert.ok(result, "in-thread fallback should still crop");
|
|
1601
|
+
const dims = extractDimensions(result);
|
|
1602
|
+
assert.ok(dims && dims.width === 5 && dims.height === 5, "fallback crop should have correct dims");
|
|
1603
|
+
} finally {
|
|
1604
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_DECODE_WORKER;
|
|
1605
|
+
else process.env.PI_VISION_PROXY_DECODE_WORKER = prev;
|
|
1606
|
+
}
|
|
1607
|
+
});
|
|
1608
|
+
|
|
1609
|
+
it("handles several sequential crops (worker pool reuse)", async () => {
|
|
1610
|
+
const png = await create10x10Png();
|
|
1611
|
+
for (let i = 0; i < 5; i++) {
|
|
1612
|
+
const result = await cropImage(png, { x: 0, y: 0, width: 5, height: 5 }, "image/png");
|
|
1613
|
+
assert.ok(result, `crop ${i} should succeed`);
|
|
1614
|
+
const dims = extractDimensions(result);
|
|
1615
|
+
assert.ok(dims && dims.width === 5 && dims.height === 5, `crop ${i} dims`);
|
|
1616
|
+
}
|
|
1617
|
+
});
|
|
1618
|
+
|
|
1619
|
+
it("works with pooling disabled (spawn-per-call)", async () => {
|
|
1620
|
+
const prev = process.env.PI_VISION_PROXY_DECODE_WORKER_POOL;
|
|
1621
|
+
process.env.PI_VISION_PROXY_DECODE_WORKER_POOL = "0";
|
|
1622
|
+
try {
|
|
1623
|
+
const png = await create10x10Png();
|
|
1624
|
+
const a = await cropImage(png, { x: 0, y: 0, width: 5, height: 5 }, "image/png");
|
|
1625
|
+
const b = await cropImage(png, { x: 1, y: 1, width: 4, height: 4 }, "image/png");
|
|
1626
|
+
assert.ok(a && b, "both crops should succeed without pooling");
|
|
1627
|
+
} finally {
|
|
1628
|
+
if (prev === undefined) delete process.env.PI_VISION_PROXY_DECODE_WORKER_POOL;
|
|
1629
|
+
else process.env.PI_VISION_PROXY_DECODE_WORKER_POOL = prev;
|
|
1630
|
+
}
|
|
1631
|
+
});
|
|
1632
|
+
|
|
1633
|
+
it("hard-terminates the worker on timeout (returns null, does not hang)", async () => {
|
|
1634
|
+
// Force pool=0 so a *fresh* worker is spawned (never a warmed pooled one).
|
|
1635
|
+
// Spinning up a thread + loading ImageScript + instantiating the WASM codec
|
|
1636
|
+
// is reliably far slower than the 1ms timeout regardless of machine speed,
|
|
1637
|
+
// so the main-thread timer fires and terminate()s the worker — proving the
|
|
1638
|
+
// timeout is a hard limit, without depending on wall-clock scheduling luck.
|
|
1639
|
+
const prevWorker = process.env.PI_VISION_PROXY_DECODE_WORKER;
|
|
1640
|
+
const prevPool = process.env.PI_VISION_PROXY_DECODE_WORKER_POOL;
|
|
1641
|
+
const prevTimeout = process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS;
|
|
1642
|
+
process.env.PI_VISION_PROXY_DECODE_WORKER = "1";
|
|
1643
|
+
process.env.PI_VISION_PROXY_DECODE_WORKER_POOL = "0";
|
|
1644
|
+
process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS = "1";
|
|
1645
|
+
try {
|
|
1646
|
+
const png = await create10x10Png();
|
|
1647
|
+
const started = Date.now();
|
|
1648
|
+
const result = await cropImage(png, { x: 0, y: 0, width: 10, height: 10 }, "image/png");
|
|
1649
|
+
assert.equal(result, null, "timed-out crop should return null");
|
|
1650
|
+
assert.ok(Date.now() - started < 5000, "should return promptly, not hang");
|
|
1651
|
+
} finally {
|
|
1652
|
+
if (prevWorker === undefined) delete process.env.PI_VISION_PROXY_DECODE_WORKER;
|
|
1653
|
+
else process.env.PI_VISION_PROXY_DECODE_WORKER = prevWorker;
|
|
1654
|
+
if (prevPool === undefined) delete process.env.PI_VISION_PROXY_DECODE_WORKER_POOL;
|
|
1655
|
+
else process.env.PI_VISION_PROXY_DECODE_WORKER_POOL = prevPool;
|
|
1656
|
+
if (prevTimeout === undefined) delete process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS;
|
|
1657
|
+
else process.env.PI_VISION_PROXY_DECODE_TIMEOUT_MS = prevTimeout;
|
|
1658
|
+
}
|
|
1659
|
+
});
|
|
1180
1660
|
});
|
|
1181
1661
|
|
|
1182
1662
|
describe("piAiImageToBuffer / bufferToPiAiImage", () => {
|
|
@@ -1780,8 +2260,9 @@ describe("Security: image decode bomb protection", () => {
|
|
|
1780
2260
|
const img = new Image(100, 100);
|
|
1781
2261
|
const encoded = Buffer.from(await img.encode(1));
|
|
1782
2262
|
const hash = "test-decode-bomb-normal";
|
|
1783
|
-
|
|
1784
|
-
|
|
2263
|
+
const store = createImageMetaStore();
|
|
2264
|
+
storeImageMeta(store, hash, encoded);
|
|
2265
|
+
const meta = store.get(hash);
|
|
1785
2266
|
// Normal image should be accepted
|
|
1786
2267
|
assert.ok(meta, "normal image should be stored");
|
|
1787
2268
|
});
|
|
@@ -1872,10 +2353,21 @@ describe("Review fixes: storeImageMeta filename backfill", () => {
|
|
|
1872
2353
|
const img = new Image(50, 60);
|
|
1873
2354
|
const encoded = Buffer.from(await img.encode(1));
|
|
1874
2355
|
const hash = "test-backfill-filename";
|
|
1875
|
-
|
|
1876
|
-
storeImageMeta(hash, encoded
|
|
1877
|
-
|
|
2356
|
+
const store = createImageMetaStore();
|
|
2357
|
+
storeImageMeta(store, hash, encoded); // first call, no filename
|
|
2358
|
+
storeImageMeta(store, hash, encoded, "photo.png"); // second call, with filename
|
|
2359
|
+
const meta = store.get(hash);
|
|
1878
2360
|
assert.ok(meta, "meta should exist");
|
|
1879
2361
|
assert.equal(meta!.filename, "photo.png", "filename should be backfilled");
|
|
1880
2362
|
});
|
|
2363
|
+
|
|
2364
|
+
it("keeps stores isolated — one session's metadata does not leak into another (issue #12)", async () => {
|
|
2365
|
+
const { Image } = await import("imagescript");
|
|
2366
|
+
const encoded = Buffer.from(await new Image(40, 30).encode(1));
|
|
2367
|
+
const sessionA = createImageMetaStore();
|
|
2368
|
+
const sessionB = createImageMetaStore();
|
|
2369
|
+
storeImageMeta(sessionA, "shared-hash", encoded, "a.png");
|
|
2370
|
+
assert.ok(sessionA.get("shared-hash"), "session A should have the metadata");
|
|
2371
|
+
assert.equal(sessionB.get("shared-hash"), undefined, "session B must not inherit it");
|
|
2372
|
+
});
|
|
1881
2373
|
});
|