quickjs-emscripten-sync 1.8.4 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +231 -150
- package/dist/index.d.ts +50 -7
- package/dist/quickjs-emscripten-sync.js +652 -387
- package/dist/quickjs-emscripten-sync.umd.cjs +17 -8
- package/package.json +5 -2
- package/src/index.test.ts +250 -5
- package/src/index.ts +126 -9
- package/src/jsonleak.test.ts +178 -0
- package/src/marshal/custom.ts +16 -2
- package/src/marshal/function.ts +6 -2
- package/src/marshal/hostref.ts +18 -0
- package/src/marshal/index.ts +25 -2
- package/src/marshal/mapset.ts +37 -0
- package/src/marshal/object.ts +4 -1
- package/src/marshal/primitive.ts +2 -11
- package/src/marshal/properties.ts +19 -6
- package/src/unmarshal/custom.ts +35 -4
- package/src/unmarshal/function.ts +2 -2
- package/src/unmarshal/hostref.ts +21 -0
- package/src/unmarshal/index.ts +21 -1
- package/src/unmarshal/mapset.ts +43 -0
- package/src/unmarshal/object.ts +16 -20
- package/src/unmarshal/primitive.ts +6 -17
- package/src/unmarshal/properties.ts +7 -6
- package/src/vmmap.ts +59 -69
- package/src/vmutil.ts +97 -12
- package/src/wrapper.ts +97 -30
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type {
|
|
|
2
2
|
QuickJSDeferredPromise,
|
|
3
3
|
QuickJSHandle,
|
|
4
4
|
QuickJSContext,
|
|
5
|
+
QuickJSAsyncContext,
|
|
5
6
|
SuccessOrFail,
|
|
6
7
|
VmCallResult,
|
|
7
8
|
Intrinsics,
|
|
@@ -11,9 +12,20 @@ import { wrapContext, QuickJSContextEx } from "./contextex";
|
|
|
11
12
|
import { defaultRegisteredObjects } from "./default";
|
|
12
13
|
import marshal from "./marshal";
|
|
13
14
|
import unmarshal from "./unmarshal";
|
|
15
|
+
import unmarshalHostRef from "./unmarshal/hostref";
|
|
14
16
|
import { complexity, isES2015Class, isObject, walkObject } from "./util";
|
|
15
17
|
import VMMap from "./vmmap";
|
|
16
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
call,
|
|
20
|
+
isHandleObject,
|
|
21
|
+
json,
|
|
22
|
+
consume,
|
|
23
|
+
consumeAll,
|
|
24
|
+
mayConsume,
|
|
25
|
+
handleFrom,
|
|
26
|
+
enableFnCache,
|
|
27
|
+
disposeFnCache,
|
|
28
|
+
} from "./vmutil";
|
|
17
29
|
import { wrap, wrapHandle, unwrap, unwrapHandle, Wrapped } from "./wrapper";
|
|
18
30
|
|
|
19
31
|
export {
|
|
@@ -53,6 +65,10 @@ export type Options = {
|
|
|
53
65
|
compat?: boolean;
|
|
54
66
|
/** Experimental: use QuickJSContextEx, which wraps existing QuickJSContext. */
|
|
55
67
|
experimentalContextEx?: boolean;
|
|
68
|
+
/** Globally enable sync mode (default `true`). When `false`, objects are not wrapped with proxies and marshalled handles are disposed after use, so `arena.sync` has no effect but objects are not retained for their whole lifetime. Useful to avoid memory growth when frequently exchanging short-lived objects. */
|
|
69
|
+
syncEnabled?: boolean;
|
|
70
|
+
/** A callback that returns whether an object should be passed to the VM by reference (as an opaque HostRef) instead of being marshalled by value/proxy. The guest cannot read such objects, but can hold them and pass them back to the host, where they resolve to the original object. */
|
|
71
|
+
marshalByReference?: (target: any) => boolean;
|
|
56
72
|
};
|
|
57
73
|
|
|
58
74
|
/**
|
|
@@ -63,6 +79,14 @@ export class Arena {
|
|
|
63
79
|
_map: VMMap;
|
|
64
80
|
_registeredMap: VMMap;
|
|
65
81
|
_registeredMapDispose = new Set<any>();
|
|
82
|
+
// Handles with no owner: "json" copies and BigInt values. Unlike
|
|
83
|
+
// proxy-marshalled objects they are not identity-tracked in `_map`. A nested
|
|
84
|
+
// one is disposed by its parent consumer via `_disposeTransient` as soon as
|
|
85
|
+
// the value has been copied; this set is the fallback that frees any that
|
|
86
|
+
// were never nested (e.g. a top-level value before its caller consumes it) on
|
|
87
|
+
// dispose. Top-level handles are removed in `_marshal` since their caller
|
|
88
|
+
// disposes them via `mayConsume`.
|
|
89
|
+
_transientHandles = new Set<QuickJSHandle>();
|
|
66
90
|
_sync = new Set<any>();
|
|
67
91
|
_temporalSync = new Set<any>();
|
|
68
92
|
_symbol = Symbol();
|
|
@@ -80,6 +104,7 @@ export class Arena {
|
|
|
80
104
|
}
|
|
81
105
|
|
|
82
106
|
this.context = options?.experimentalContextEx ? wrapContext(ctx) : ctx;
|
|
107
|
+
enableFnCache(this.context);
|
|
83
108
|
this._options = options;
|
|
84
109
|
this._symbolHandle = ctx.unwrapResult(ctx.evalCode(`Symbol()`));
|
|
85
110
|
this._map = new VMMap(ctx);
|
|
@@ -91,12 +116,22 @@ export class Arena {
|
|
|
91
116
|
* Dispose of the arena and managed handles. This method won't dispose the VM itself, so the VM has to be disposed of manually.
|
|
92
117
|
*/
|
|
93
118
|
dispose() {
|
|
119
|
+
for (const h of this._transientHandles) {
|
|
120
|
+
if (h.alive) h.dispose();
|
|
121
|
+
}
|
|
122
|
+
this._transientHandles.clear();
|
|
94
123
|
this._map.dispose();
|
|
95
124
|
this._registeredMap.dispose();
|
|
96
125
|
this._symbolHandle.dispose();
|
|
126
|
+
disposeFnCache(this.context);
|
|
97
127
|
this.context.disposeEx?.();
|
|
98
128
|
}
|
|
99
129
|
|
|
130
|
+
/** Allows `using arena = new Arena(...)` to dispose the arena automatically. */
|
|
131
|
+
[Symbol.dispose]() {
|
|
132
|
+
this.dispose();
|
|
133
|
+
}
|
|
134
|
+
|
|
100
135
|
/**
|
|
101
136
|
* Evaluate JS code in the VM and get the result as an object on the host side. It also converts and re-throws error objects when an error is thrown during evaluation.
|
|
102
137
|
*/
|
|
@@ -150,7 +185,7 @@ export class Arena {
|
|
|
150
185
|
if ("value" in result) {
|
|
151
186
|
return result.value;
|
|
152
187
|
}
|
|
153
|
-
throw this._unwrapIfNotSynced(result.error
|
|
188
|
+
throw this._unwrapIfNotSynced(consume(result.error, this._unmarshal));
|
|
154
189
|
}
|
|
155
190
|
|
|
156
191
|
/**
|
|
@@ -355,12 +390,12 @@ export class Arena {
|
|
|
355
390
|
if ("value" in result) {
|
|
356
391
|
return result.value;
|
|
357
392
|
}
|
|
358
|
-
throw this._unwrapIfNotSynced(result.error
|
|
393
|
+
throw this._unwrapIfNotSynced(consume(result.error, this._unmarshal));
|
|
359
394
|
}
|
|
360
395
|
|
|
361
396
|
_unwrapResultAndUnmarshal(result: VmCallResult<QuickJSHandle> | undefined): any {
|
|
362
397
|
if (!result) return;
|
|
363
|
-
return this._unwrapIfNotSynced(this._unwrapResult(result)
|
|
398
|
+
return this._unwrapIfNotSynced(consume(this._unwrapResult(result), this._unmarshal));
|
|
364
399
|
}
|
|
365
400
|
|
|
366
401
|
_isMarshalable = (t: unknown): boolean | "json" => {
|
|
@@ -368,6 +403,19 @@ export class Arena {
|
|
|
368
403
|
return (typeof im === "function" ? im(this._unwrap(t)) : im) ?? "json";
|
|
369
404
|
};
|
|
370
405
|
|
|
406
|
+
_marshalByReference = (t: unknown): boolean => {
|
|
407
|
+
return !!this._options?.marshalByReference?.(this._unwrap(t));
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
// Register an opaque HostRef handle by its host value without wrapping it.
|
|
411
|
+
_registerHostRef = (t: unknown, handle: QuickJSHandle): QuickJSHandle => {
|
|
412
|
+
const u = this._unwrap(t);
|
|
413
|
+
const existing = this._map.get(u);
|
|
414
|
+
if (existing) return existing;
|
|
415
|
+
this._map.set(u, handle);
|
|
416
|
+
return handle;
|
|
417
|
+
};
|
|
418
|
+
|
|
371
419
|
_marshalFind = (t: unknown) => {
|
|
372
420
|
const unwrappedT = this._unwrap(t);
|
|
373
421
|
const handle =
|
|
@@ -383,8 +431,29 @@ export class Arena {
|
|
|
383
431
|
h: QuickJSHandle | QuickJSDeferredPromise,
|
|
384
432
|
mode: true | "json" | undefined,
|
|
385
433
|
): Wrapped<QuickJSHandle> | undefined => {
|
|
386
|
-
if (mode === "json")
|
|
387
|
-
|
|
434
|
+
if (mode === "json") {
|
|
435
|
+
// json handles have no identity to track; register as transient so they
|
|
436
|
+
// are disposed once consumed (or on dispose) instead of leaking.
|
|
437
|
+
this._registerTransient(handleFrom(h));
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
const registered = this._register(t, handleFrom(h), this._map);
|
|
441
|
+
if (registered) return registered[1];
|
|
442
|
+
// `_register` bails for value-only built-ins that `_wrap` excludes (Map,
|
|
443
|
+
// Set, Date, ArrayBuffer, TypedArray): they are marshalled by value with no
|
|
444
|
+
// entry in `_map`, so their handle has no owner. Track it as transient so a
|
|
445
|
+
// nested one is disposed once consumed, like the json path. (Objects already
|
|
446
|
+
// owned by `_registeredMap` are reached via `_marshalFind`, not here.)
|
|
447
|
+
if (!this._registeredMap.has(t)) this._registerTransient(handleFrom(h));
|
|
448
|
+
return;
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
_registerTransient = (handle: QuickJSHandle): void => {
|
|
452
|
+
this._transientHandles.add(handle);
|
|
453
|
+
};
|
|
454
|
+
|
|
455
|
+
_disposeTransient = (handle: QuickJSHandle): void => {
|
|
456
|
+
if (this._transientHandles.delete(handle) && handle.alive) handle.dispose();
|
|
388
457
|
};
|
|
389
458
|
|
|
390
459
|
_marshalPreApply = (target: (...args: any[]) => any, that: unknown, args: unknown[]): void => {
|
|
@@ -405,21 +474,35 @@ export class Arena {
|
|
|
405
474
|
return [registered, false];
|
|
406
475
|
}
|
|
407
476
|
|
|
408
|
-
|
|
477
|
+
// Pass-by-reference objects must reference the original, not a proxy wrapper.
|
|
478
|
+
const marshalTarget = this._marshalByReference(target)
|
|
479
|
+
? this._unwrap(target)
|
|
480
|
+
: this._wrap(target) ?? target;
|
|
481
|
+
|
|
482
|
+
const handle = marshal(marshalTarget, {
|
|
409
483
|
ctx: this.context,
|
|
410
484
|
unmarshal: this._unmarshal,
|
|
411
485
|
isMarshalable: this._isMarshalable,
|
|
486
|
+
marshalByReference: this._marshalByReference,
|
|
487
|
+
registerHostRef: this._registerHostRef,
|
|
412
488
|
find: this._marshalFind,
|
|
413
489
|
pre: this._marshalPre,
|
|
490
|
+
registerTransient: this._registerTransient,
|
|
491
|
+
disposeTransient: this._disposeTransient,
|
|
414
492
|
preApply: this._marshalPreApply,
|
|
415
493
|
custom: this._options?.customMarshaller,
|
|
416
494
|
});
|
|
417
495
|
|
|
418
|
-
|
|
496
|
+
// A top-level transient handle is disposed by the caller via `mayConsume`,
|
|
497
|
+
// so it must not also be retained (and re-disposed) by the transient set.
|
|
498
|
+
this._transientHandles.delete(handle);
|
|
499
|
+
|
|
500
|
+
const syncEnabled = this._options?.syncEnabled ?? true;
|
|
501
|
+
return [handle, !syncEnabled || !this._map.hasHandle(handle)];
|
|
419
502
|
};
|
|
420
503
|
|
|
421
504
|
_preUnmarshal = (t: any, h: QuickJSHandle): Wrapped<any> => {
|
|
422
|
-
return this._register(t, h, undefined, true)?.[0];
|
|
505
|
+
return this._register(t, h, undefined, this._options?.syncEnabled ?? true)?.[0];
|
|
423
506
|
};
|
|
424
507
|
|
|
425
508
|
_unmarshalFind = (h: QuickJSHandle): unknown => {
|
|
@@ -432,6 +515,12 @@ export class Arena {
|
|
|
432
515
|
return registered;
|
|
433
516
|
}
|
|
434
517
|
|
|
518
|
+
// Resolve opaque HostRefs before wrapping, since a proxy would hide them.
|
|
519
|
+
if (this._options?.marshalByReference) {
|
|
520
|
+
const ref = unmarshalHostRef(this.context, handle);
|
|
521
|
+
if (ref) return ref.value;
|
|
522
|
+
}
|
|
523
|
+
|
|
435
524
|
const [wrappedHandle] = this._wrapHandle(handle);
|
|
436
525
|
return unmarshal(wrappedHandle ?? handle, {
|
|
437
526
|
ctx: this.context,
|
|
@@ -439,6 +528,7 @@ export class Arena {
|
|
|
439
528
|
find: this._unmarshalFind,
|
|
440
529
|
pre: this._preUnmarshal,
|
|
441
530
|
custom: this._options?.customUnmarshaller,
|
|
531
|
+
hostRef: !!this._options?.marshalByReference,
|
|
442
532
|
});
|
|
443
533
|
};
|
|
444
534
|
|
|
@@ -487,6 +577,7 @@ export class Arena {
|
|
|
487
577
|
this._marshal,
|
|
488
578
|
this._syncMode,
|
|
489
579
|
this._options?.isWrappable,
|
|
580
|
+
this._options?.syncEnabled ?? true,
|
|
490
581
|
);
|
|
491
582
|
}
|
|
492
583
|
|
|
@@ -508,6 +599,7 @@ export class Arena {
|
|
|
508
599
|
this._unmarshal,
|
|
509
600
|
this._syncMode,
|
|
510
601
|
this._options?.isHandleWrappable,
|
|
602
|
+
this._options?.syncEnabled ?? true,
|
|
511
603
|
);
|
|
512
604
|
}
|
|
513
605
|
|
|
@@ -515,3 +607,28 @@ export class Arena {
|
|
|
515
607
|
return unwrapHandle(this.context, target, this._symbolHandle);
|
|
516
608
|
}
|
|
517
609
|
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* An Arena backed by a {@link QuickJSAsyncContext}. In addition to everything
|
|
613
|
+
* `Arena` offers, it can evaluate code asynchronously with `evalCodeAsync`,
|
|
614
|
+
* which lets the VM await host promises (e.g. async module loaders or async
|
|
615
|
+
* functions exposed from the host) without manually pumping pending jobs.
|
|
616
|
+
*/
|
|
617
|
+
export class AsyncArena extends Arena {
|
|
618
|
+
asyncContext: QuickJSAsyncContext;
|
|
619
|
+
|
|
620
|
+
constructor(ctx: QuickJSAsyncContext, options?: Options) {
|
|
621
|
+
super(ctx, options);
|
|
622
|
+
this.asyncContext = ctx;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Evaluate JS code asynchronously in the VM and get the result on the host
|
|
627
|
+
* side. Like `evalCode`, it converts and re-throws errors thrown during
|
|
628
|
+
* evaluation. Use this when the code awaits host-provided promises.
|
|
629
|
+
*/
|
|
630
|
+
async evalCodeAsync<T = any>(code: string, filename?: string): Promise<T> {
|
|
631
|
+
const result = await this.asyncContext.evalCodeAsync(code, filename);
|
|
632
|
+
return this._unwrapResultAndUnmarshal(result);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import variant from "@jitl/quickjs-wasmfile-debug-sync";
|
|
2
|
+
import { newQuickJSWASMModuleFromVariant } from "quickjs-emscripten";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import { Arena } from ".";
|
|
6
|
+
|
|
7
|
+
// A non-plain object that defaultIsMarshalable would reject, forcing the
|
|
8
|
+
// "json" marshal path.
|
|
9
|
+
class Weird {
|
|
10
|
+
x = 1;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const jsonIsMarshalable = (t: unknown) =>
|
|
14
|
+
typeof t !== "object" || t === null || Object.getPrototypeOf(t) === Object.prototype
|
|
15
|
+
? true
|
|
16
|
+
: "json";
|
|
17
|
+
|
|
18
|
+
async function withArena(
|
|
19
|
+
options: ConstructorParameters<typeof Arena>[1],
|
|
20
|
+
fn: (arena: Arena) => void,
|
|
21
|
+
) {
|
|
22
|
+
const mod = await newQuickJSWASMModuleFromVariant(variant as any);
|
|
23
|
+
const ctx = mod.newContext();
|
|
24
|
+
const arena = new Arena(ctx, options);
|
|
25
|
+
fn(arena);
|
|
26
|
+
arena.dispose();
|
|
27
|
+
// The debug-sync runtime aborts on dispose if any GC object handle leaked, so
|
|
28
|
+
// simply reaching here without an Emscripten abort means no leak.
|
|
29
|
+
expect(() => ctx.dispose()).not.toThrow();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe("json marshal path handle leak", () => {
|
|
33
|
+
it("does not leak when a nested value is marshalled via json", async () => {
|
|
34
|
+
await withArena({ isMarshalable: jsonIsMarshalable }, arena => {
|
|
35
|
+
// Outer object goes through the proxy/object path; the nested `weird`
|
|
36
|
+
// property resolves to "json", creating a JSON.parse handle that nobody
|
|
37
|
+
// would otherwise dispose.
|
|
38
|
+
arena.expose({ obj: { weird: new Weird() } });
|
|
39
|
+
expect(arena.evalCode(`obj.weird.x`)).toBe(1);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("does not leak nor double-dispose a top-level json value", async () => {
|
|
44
|
+
await withArena({ isMarshalable: jsonIsMarshalable }, arena => {
|
|
45
|
+
// host fn receives a top-level json value as an argument; the handle is
|
|
46
|
+
// disposed by mayConsume and must not be re-disposed on dispose().
|
|
47
|
+
let received: any;
|
|
48
|
+
arena.expose({ sink: (v: any) => (received = v) });
|
|
49
|
+
arena.evalCode(`sink`);
|
|
50
|
+
const fn = arena.evalCode(`sink`);
|
|
51
|
+
fn(new Weird());
|
|
52
|
+
expect(received).toEqual({ x: 1 });
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("does not leak with the default marshal mode (json)", async () => {
|
|
57
|
+
// With no isMarshalable option the default mode is "json", so even a plain
|
|
58
|
+
// nested object exercises the json path that used to leak. (Functions are
|
|
59
|
+
// dropped by json serialization, so this path only carries plain values.)
|
|
60
|
+
await withArena(undefined, arena => {
|
|
61
|
+
arena.expose({ data: { a: { b: 1 } } });
|
|
62
|
+
expect(arena.evalCode(`data.a.b`)).toBe(1);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("does not leak when a json value round-trips VM -> host", async () => {
|
|
67
|
+
await withArena({ isMarshalable: jsonIsMarshalable }, arena => {
|
|
68
|
+
const received: any[] = [];
|
|
69
|
+
arena.expose({ host: { f: (v: any) => received.push(v) } });
|
|
70
|
+
// VM builds a non-plain object and passes it to the host callback.
|
|
71
|
+
arena.evalCode(`
|
|
72
|
+
const o = Object.create({ tag: "proto" });
|
|
73
|
+
o.a = 1;
|
|
74
|
+
host.f({ nested: o });
|
|
75
|
+
`);
|
|
76
|
+
expect(received).toEqual([{ nested: { a: 1 } }]);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
describe("value-marshalled built-ins do not leak when nested", () => {
|
|
82
|
+
// Map/Set/Date/ArrayBuffer/TypedArray are excluded from proxy wrapping and
|
|
83
|
+
// marshalled by value, so they get no entry in `_map`. A top-level one is
|
|
84
|
+
// disposed by mayConsume, but a nested one used to have no owner and leaked
|
|
85
|
+
// its handle (aborting the debug runtime on dispose).
|
|
86
|
+
const builtinMarshalable = (t: unknown) =>
|
|
87
|
+
t instanceof Map || t instanceof Set ? true : jsonIsMarshalable(t);
|
|
88
|
+
|
|
89
|
+
it("nested Map", async () => {
|
|
90
|
+
await withArena({ isMarshalable: builtinMarshalable }, arena => {
|
|
91
|
+
arena.expose({ data: { m: new Map<string, unknown>([["b", new Weird()]]) } });
|
|
92
|
+
expect(arena.evalCode(`data.m.get("b").x`)).toBe(1);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("nested Set", async () => {
|
|
97
|
+
await withArena({ isMarshalable: builtinMarshalable }, arena => {
|
|
98
|
+
arena.expose({ data: { s: new Set<number>([1, 2]) } });
|
|
99
|
+
expect(arena.evalCode(`data.s.size`)).toBe(2);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("nested Date and TypedArray", async () => {
|
|
104
|
+
await withArena({ isMarshalable: true }, arena => {
|
|
105
|
+
arena.expose({ data: { d: new Date(0), a: new Uint8Array([1, 2, 3]) } });
|
|
106
|
+
expect(arena.evalCode(`data.a[1]`)).toBe(2);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("deeply nested Map", async () => {
|
|
111
|
+
await withArena({ isMarshalable: builtinMarshalable }, arena => {
|
|
112
|
+
arena.expose({ data: { a: { b: { m: new Map<string, number>([["x", 9]]) } } } });
|
|
113
|
+
expect(arena.evalCode(`data.a.b.m.get("x")`)).toBe(9);
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe("transient handles are freed as they are consumed (no accumulation)", () => {
|
|
119
|
+
// Marshal `make(i)` many times, disposing the top-level handle each time
|
|
120
|
+
// (mirroring `mayConsume`). With syncEnabled:false nothing is retained in
|
|
121
|
+
// `_map`, so any growth in live VM memory / object count is a genuine leak of
|
|
122
|
+
// a nested transient (json copy or BigInt) that was not disposed.
|
|
123
|
+
async function liveGrowth(
|
|
124
|
+
make: (i: number) => unknown,
|
|
125
|
+
isMarshalable: ConstructorParameters<typeof Arena>[1] extends infer O
|
|
126
|
+
? O extends { isMarshalable?: infer M }
|
|
127
|
+
? M
|
|
128
|
+
: never
|
|
129
|
+
: never,
|
|
130
|
+
) {
|
|
131
|
+
const mod = await newQuickJSWASMModuleFromVariant(variant as any);
|
|
132
|
+
const ctx = mod.newContext();
|
|
133
|
+
const arena = new Arena(ctx, { isMarshalable, syncEnabled: false }) as any;
|
|
134
|
+
const mem = () => ctx.runtime.computeMemoryUsage().consume((h: any) => ctx.dump(h));
|
|
135
|
+
const marshalOne = (i: number) => {
|
|
136
|
+
// `_marshal` returns [handle, shouldBeDisposed]; honour it like mayConsume.
|
|
137
|
+
const [h, shouldDispose] = arena._marshal(make(i));
|
|
138
|
+
if (shouldDispose && h.alive) h.dispose();
|
|
139
|
+
};
|
|
140
|
+
// Warm up so one-time shape/atom allocations are not counted.
|
|
141
|
+
for (let i = 0; i < 30; i++) marshalOne(i);
|
|
142
|
+
const before = mem();
|
|
143
|
+
for (let i = 0; i < 200; i++) marshalOne(i + 100000);
|
|
144
|
+
const after = mem();
|
|
145
|
+
arena.dispose();
|
|
146
|
+
ctx.dispose();
|
|
147
|
+
return {
|
|
148
|
+
usedBytes: after.memory_used_size - before.memory_used_size,
|
|
149
|
+
objects: after.obj_count - before.obj_count,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
it("does not leak nested BigInt values", async () => {
|
|
154
|
+
// Pre-fix this leaked ~7 bytes of BigInt storage per iteration.
|
|
155
|
+
const { usedBytes } = await liveGrowth(i => ({ b: BigInt(i) * 99999999n }), true);
|
|
156
|
+
expect(usedBytes).toBeLessThan(400);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("does not accumulate nested json objects", async () => {
|
|
160
|
+
// Pre-fix each iteration retained one JSON.parse object until dispose.
|
|
161
|
+
const { objects } = await liveGrowth(() => ({ w: new Weird() }), jsonIsMarshalable);
|
|
162
|
+
expect(objects).toBeLessThan(20);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("frees json/BigInt values stored in a Map or Set", async () => {
|
|
166
|
+
// A top-level Map/Set so the collection handle itself is not retained; this
|
|
167
|
+
// isolates the lifetime of its BigInt/json entries, which must be freed.
|
|
168
|
+
const collMarshalable = (t: unknown) =>
|
|
169
|
+
t instanceof Map || t instanceof Set ? true : jsonIsMarshalable(t);
|
|
170
|
+
const map = await liveGrowth(
|
|
171
|
+
i => new Map<string, unknown>([["a", BigInt(i)], ["b", new Weird()]]),
|
|
172
|
+
collMarshalable,
|
|
173
|
+
);
|
|
174
|
+
expect(map.usedBytes).toBeLessThan(400);
|
|
175
|
+
const set = await liveGrowth(() => new Set<unknown>([new Weird()]), collMarshalable);
|
|
176
|
+
expect(set.usedBytes).toBeLessThan(400);
|
|
177
|
+
});
|
|
178
|
+
});
|
package/src/marshal/custom.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { QuickJSContext, QuickJSHandle } from "quickjs-emscripten";
|
|
2
2
|
|
|
3
|
-
import { call } from "../vmutil";
|
|
3
|
+
import { call, consumeAll } from "../vmutil";
|
|
4
4
|
|
|
5
5
|
export default function marshalCustom(
|
|
6
6
|
ctx: QuickJSContext,
|
|
@@ -33,4 +33,18 @@ export function date(target: unknown, ctx: QuickJSContext): QuickJSHandle | unde
|
|
|
33
33
|
return handle;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
export
|
|
36
|
+
export function arrayBuffer(target: unknown, ctx: QuickJSContext): QuickJSHandle | undefined {
|
|
37
|
+
if (target instanceof ArrayBuffer) {
|
|
38
|
+
return ctx.newArrayBuffer(target.slice(0));
|
|
39
|
+
}
|
|
40
|
+
if (ArrayBuffer.isView(target)) {
|
|
41
|
+
// TypedArray or DataView: copy the viewed bytes and rebuild in the VM.
|
|
42
|
+
const bytes = new Uint8Array(target.buffer, target.byteOffset, target.byteLength).slice();
|
|
43
|
+
return consumeAll(
|
|
44
|
+
[ctx.newArrayBuffer(bytes.buffer), ctx.newString(target.constructor.name)],
|
|
45
|
+
([buf, name]) => call(ctx, `(buf, name) => new globalThis[name](buf)`, undefined, buf, name),
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const defaultCustom = [symbol, date, arrayBuffer];
|
package/src/marshal/function.ts
CHANGED
|
@@ -12,6 +12,7 @@ export default function marshalFunction(
|
|
|
12
12
|
unmarshal: (handle: QuickJSHandle) => unknown,
|
|
13
13
|
preMarshal: (target: unknown, handle: QuickJSHandle) => QuickJSHandle | undefined,
|
|
14
14
|
preApply?: (target: (...args: any[]) => any, thisArg: unknown, args: unknown[]) => any,
|
|
15
|
+
disposeTransient: (handle: QuickJSHandle) => void = () => {},
|
|
15
16
|
): QuickJSHandle | undefined {
|
|
16
17
|
if (typeof target !== "function") return;
|
|
17
18
|
|
|
@@ -24,7 +25,10 @@ export default function marshalFunction(
|
|
|
24
25
|
// Class constructors cannot be invoked without new expression, and new.target is not changed
|
|
25
26
|
const result = new target(...args);
|
|
26
27
|
Object.entries(result).forEach(([key, value]) => {
|
|
27
|
-
|
|
28
|
+
const valueHandle = marshal(value);
|
|
29
|
+
ctx.setProp(this, key, valueHandle);
|
|
30
|
+
// setProp dup'd the value into `this`; drop ours if it was transient.
|
|
31
|
+
disposeTransient(valueHandle);
|
|
28
32
|
});
|
|
29
33
|
return this;
|
|
30
34
|
}
|
|
@@ -47,7 +51,7 @@ export default function marshalFunction(
|
|
|
47
51
|
);
|
|
48
52
|
|
|
49
53
|
const handle = preMarshal(target, raw) ?? raw;
|
|
50
|
-
marshalProperties(ctx, target, raw, marshal);
|
|
54
|
+
marshalProperties(ctx, target, raw, marshal, disposeTransient);
|
|
51
55
|
|
|
52
56
|
return handle;
|
|
53
57
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { QuickJSContext, QuickJSHandle } from "quickjs-emscripten";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Marshal a host object as an opaque HostRef handle instead of deep-copying or
|
|
5
|
+
* proxying it. The guest cannot read the object, but can hold it and pass it
|
|
6
|
+
* back to the host, where it unmarshals to the original object by reference.
|
|
7
|
+
*/
|
|
8
|
+
export default function marshalHostRef(
|
|
9
|
+
ctx: QuickJSContext,
|
|
10
|
+
target: unknown,
|
|
11
|
+
// Registers handle <-> target without wrapping (a proxy would hide the opaque
|
|
12
|
+
// HostRef). Keeps identity and lets the arena dispose the handle on teardown.
|
|
13
|
+
register: (target: unknown, handle: QuickJSHandle) => QuickJSHandle,
|
|
14
|
+
): QuickJSHandle | undefined {
|
|
15
|
+
if (typeof target !== "object" && typeof target !== "function") return;
|
|
16
|
+
if (target === null) return;
|
|
17
|
+
return register(target, ctx.newHostRef(target as object).handle);
|
|
18
|
+
}
|
package/src/marshal/index.ts
CHANGED
|
@@ -2,7 +2,9 @@ import type { QuickJSDeferredPromise, QuickJSHandle, QuickJSContext } from "quic
|
|
|
2
2
|
|
|
3
3
|
import marshalCustom, { defaultCustom } from "./custom";
|
|
4
4
|
import marshalFunction from "./function";
|
|
5
|
+
import marshalHostRef from "./hostref";
|
|
5
6
|
import marshalJSON from "./json";
|
|
7
|
+
import marshalMapSet from "./mapset";
|
|
6
8
|
import marshalObject from "./object";
|
|
7
9
|
import marshalPrimitive from "./primitive";
|
|
8
10
|
import marshalPromise from "./promise";
|
|
@@ -11,7 +13,15 @@ export type Options = {
|
|
|
11
13
|
ctx: QuickJSContext;
|
|
12
14
|
unmarshal: (handle: QuickJSHandle) => unknown;
|
|
13
15
|
isMarshalable?: (target: unknown) => boolean | "json";
|
|
16
|
+
marshalByReference?: (target: unknown) => boolean;
|
|
17
|
+
registerHostRef?: (target: unknown, handle: QuickJSHandle) => QuickJSHandle;
|
|
14
18
|
find: (target: unknown) => QuickJSHandle | undefined;
|
|
19
|
+
// Track a handle that nobody else owns (json copies, BigInt) so it can be
|
|
20
|
+
// disposed once consumed.
|
|
21
|
+
registerTransient?: (handle: QuickJSHandle) => void;
|
|
22
|
+
// Dispose a handle previously registered as transient, right after a parent
|
|
23
|
+
// consumer has copied it into the parent value. A no-op for owned handles.
|
|
24
|
+
disposeTransient?: (handle: QuickJSHandle) => void;
|
|
15
25
|
pre: (
|
|
16
26
|
target: unknown,
|
|
17
27
|
handle: QuickJSHandle | QuickJSDeferredPromise,
|
|
@@ -27,6 +37,10 @@ export function marshal(target: unknown, options: Options): QuickJSHandle {
|
|
|
27
37
|
{
|
|
28
38
|
const primitive = marshalPrimitive(ctx, target);
|
|
29
39
|
if (primitive) {
|
|
40
|
+
// BigInt handles are heap-allocated and, unlike strings and numbers, are
|
|
41
|
+
// not reclaimed unless explicitly disposed. Track them as transient so a
|
|
42
|
+
// nested one is freed by its parent consumer (or on dispose as a fallback).
|
|
43
|
+
if (typeof target === "bigint") options.registerTransient?.(primitive);
|
|
30
44
|
return primitive;
|
|
31
45
|
}
|
|
32
46
|
}
|
|
@@ -36,6 +50,13 @@ export function marshal(target: unknown, options: Options): QuickJSHandle {
|
|
|
36
50
|
if (handle) return handle;
|
|
37
51
|
}
|
|
38
52
|
|
|
53
|
+
// Opt-in pass-by-reference: hand the object to the VM as an opaque HostRef
|
|
54
|
+
// instead of marshalling its contents. Bypasses isMarshalable on purpose.
|
|
55
|
+
if (options.marshalByReference?.(target) && options.registerHostRef) {
|
|
56
|
+
const handle = marshalHostRef(ctx, target, options.registerHostRef);
|
|
57
|
+
if (handle) return handle;
|
|
58
|
+
}
|
|
59
|
+
|
|
39
60
|
const marshalable = isMarshalable?.(target);
|
|
40
61
|
if (marshalable === false) {
|
|
41
62
|
return ctx.undefined;
|
|
@@ -48,11 +69,13 @@ export function marshal(target: unknown, options: Options): QuickJSHandle {
|
|
|
48
69
|
}
|
|
49
70
|
|
|
50
71
|
const marshal2 = (t: unknown) => marshal(t, options);
|
|
72
|
+
const disposeTransient = options.disposeTransient;
|
|
51
73
|
return (
|
|
52
74
|
marshalCustom(ctx, target, pre2, [...defaultCustom, ...(options.custom ?? [])]) ??
|
|
53
75
|
marshalPromise(ctx, target, marshal2, pre2) ??
|
|
54
|
-
marshalFunction(ctx, target, marshal2, unmarshal, pre2, options.preApply) ??
|
|
55
|
-
|
|
76
|
+
marshalFunction(ctx, target, marshal2, unmarshal, pre2, options.preApply, disposeTransient) ??
|
|
77
|
+
marshalMapSet(ctx, target, marshal2, pre2, disposeTransient) ??
|
|
78
|
+
marshalObject(ctx, target, marshal2, pre2, disposeTransient) ??
|
|
56
79
|
ctx.undefined
|
|
57
80
|
);
|
|
58
81
|
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { QuickJSContext, QuickJSHandle } from "quickjs-emscripten";
|
|
2
|
+
|
|
3
|
+
import { call } from "../vmutil";
|
|
4
|
+
|
|
5
|
+
export default function marshalMapSet(
|
|
6
|
+
ctx: QuickJSContext,
|
|
7
|
+
target: unknown,
|
|
8
|
+
marshal: (target: unknown) => QuickJSHandle,
|
|
9
|
+
preMarshal: (target: unknown, handle: QuickJSHandle) => QuickJSHandle | undefined,
|
|
10
|
+
disposeTransient: (handle: QuickJSHandle) => void = () => {},
|
|
11
|
+
): QuickJSHandle | undefined {
|
|
12
|
+
if (target instanceof Map) {
|
|
13
|
+
const raw = call(ctx, "() => new Map()");
|
|
14
|
+
const handle = preMarshal(target, raw) ?? raw;
|
|
15
|
+
for (const [k, v] of target) {
|
|
16
|
+
const kh = marshal(k);
|
|
17
|
+
const vh = marshal(v);
|
|
18
|
+
call(ctx, "(m, k, v) => m.set(k, v)", undefined, raw, kh, vh).dispose();
|
|
19
|
+
// set() has taken its own references; drop ours if they were transient.
|
|
20
|
+
disposeTransient(kh);
|
|
21
|
+
disposeTransient(vh);
|
|
22
|
+
}
|
|
23
|
+
return handle;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if (target instanceof Set) {
|
|
27
|
+
const raw = call(ctx, "() => new Set()");
|
|
28
|
+
const handle = preMarshal(target, raw) ?? raw;
|
|
29
|
+
for (const v of target) {
|
|
30
|
+
const vh = marshal(v);
|
|
31
|
+
call(ctx, "(s, v) => s.add(v)", undefined, raw, vh).dispose();
|
|
32
|
+
// add() has taken its own reference; drop ours if it was transient.
|
|
33
|
+
disposeTransient(vh);
|
|
34
|
+
}
|
|
35
|
+
return handle;
|
|
36
|
+
}
|
|
37
|
+
}
|
package/src/marshal/object.ts
CHANGED
|
@@ -9,6 +9,7 @@ export default function marshalObject(
|
|
|
9
9
|
target: unknown,
|
|
10
10
|
marshal: (target: unknown) => QuickJSHandle,
|
|
11
11
|
preMarshal: (target: unknown, handle: QuickJSHandle) => QuickJSHandle | undefined,
|
|
12
|
+
disposeTransient: (handle: QuickJSHandle) => void = () => {},
|
|
12
13
|
): QuickJSHandle | undefined {
|
|
13
14
|
if (typeof target !== "object" || target === null) return;
|
|
14
15
|
|
|
@@ -23,9 +24,11 @@ export default function marshalObject(
|
|
|
23
24
|
: undefined;
|
|
24
25
|
if (prototypeHandle) {
|
|
25
26
|
call(ctx, "Object.setPrototypeOf", undefined, handle, prototypeHandle).dispose();
|
|
27
|
+
// setPrototypeOf has taken its own reference; drop ours if it was transient.
|
|
28
|
+
disposeTransient(prototypeHandle);
|
|
26
29
|
}
|
|
27
30
|
|
|
28
|
-
marshalProperties(ctx, target, raw, marshal);
|
|
31
|
+
marshalProperties(ctx, target, raw, marshal, disposeTransient);
|
|
29
32
|
|
|
30
33
|
return handle;
|
|
31
34
|
}
|
package/src/marshal/primitive.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { QuickJSContext, QuickJSHandle } from "quickjs-emscripten";
|
|
2
2
|
|
|
3
|
-
// import { call } from "../vmutil";
|
|
4
|
-
|
|
5
3
|
export default function marshalPrimitive(
|
|
6
4
|
ctx: QuickJSContext,
|
|
7
5
|
target: unknown,
|
|
@@ -15,17 +13,10 @@ export default function marshalPrimitive(
|
|
|
15
13
|
return ctx.newString(target);
|
|
16
14
|
case "boolean":
|
|
17
15
|
return target ? ctx.true : ctx.false;
|
|
16
|
+
case "bigint":
|
|
17
|
+
return ctx.newBigInt(target);
|
|
18
18
|
case "object":
|
|
19
19
|
return target === null ? ctx.null : undefined;
|
|
20
|
-
|
|
21
|
-
// BigInt is not supported by quickjs-emscripten
|
|
22
|
-
// case "bigint":
|
|
23
|
-
// return call(
|
|
24
|
-
// ctx,
|
|
25
|
-
// `s => BigInt(s)`,
|
|
26
|
-
// undefined,
|
|
27
|
-
// ctx.newString(target.toString())
|
|
28
|
-
// );
|
|
29
20
|
}
|
|
30
21
|
|
|
31
22
|
return undefined;
|