querysub 0.2.0

package/src/test/test.tsx

@@ -0,0 +1,552 @@
+// import { disableMeasurements } from "socket-function/src/profiling/measure";
+// disableMeasurements();
+
+import { isNode, list } from "socket-function/src/misc";
+import { logErrors } from "../errors";
+import { watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadController";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { red } from "socket-function/src/formatting/logColors";
+import { PermissionsCheck } from "../4-querysub/permissions";
+import { isDeploy } from "../3-path-functions/deployCheck";
+import { formatTime } from "socket-function/src/formatting/format";
+
+import { PromiseObj } from "../promise";
+import { Benchmark } from "../diagnostics/benchmark";
+import { formatStats, percent } from "socket-function/src/profiling/statsFormat";
+import { cache } from "socket-function/src/caching";
+import { atomic, atomicObjectWrite, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { fastHash } from "../misc/hash";
+import { hslToRGB } from "../misc/color";
+import { sat_data, sat_functions } from "../diagnostics/satSchema";
+import { qreact } from "../4-dom/qreact";
+
+
+// NOTE: In our first test of PathValueSerializer it is about 10X faster than JSONLACKS in parsing.
+//  - It will probably be even faster if we do lazy value deserialization
+
+// async function test() {
+//     let testFile = "C:/Users/quent/shard/backups/querysub.com-4/PathFunctionRunner/lkddfcwz.lkddd8fv.lj9juidm.8b1s28.0.__gACTgA3gEFk1spi.YMm6Zg1AEejUtImU.json";
+//     //testFile = "C:/Users/quent/shard/backups/1-querysub.com/PathFunctionRunner/lj1z7won.lj1z5rrz.lj1z3oe0.2zy9m.0.json";
+
+//     await measureCode(async function test() {
+//         JSONLACKS.EXTENDED_PARSER = false;
+
+//         const testBuffer = measureBlock(function readFile() { return fs.readFileSync(testFile); });
+//         let objs = await measureBlock(async function JSONLACKSParse() { return await JSONLACKS.parseLines(testBuffer) as PathValue[]; });
+
+//         let newBuffer = await pathValueSerializer.serialize(objs, { noLocks: false, compress: true, });
+//         let objs2 = await pathValueSerializer.deserialize(newBuffer, {});
+
+//         console.log(`OLD ${formatNumber(testBuffer.length)} bytes, ${formatNumber(objs.length)} objects`);
+//         console.log(`NEW ${formatNumber(newBuffer.map(x => x.length).reduce((a, b) => a + b, 0))} bytes, ${formatNumber(objs2.length)} objects`);
+
+//         // for (let obj of objs2) {
+//         //     pathValueSerializer.getPathValue(obj);
+//         // }
+
+//         // ALSO, make sure when we re-serialize, the second serialize is faster, because of the lazy objects (we will have to stop
+//         //      getting all values first...)
+//         /*
+//         let newBuffer2 = await pathValueSerializer.serialize(objs2, { noLocks: true });
+//         let objs3 = await pathValueSerializer.deserialize(newBuffer2);
+//         console.log(`NEW2 ${formatNumber(newBuffer2.map(x => x.length).reduce((a, b) => a + b, 0))} bytes, ${formatNumber(objs3.length)} objects`);
+//         for (let obj of objs3) {
+//             pathValueSerializer.getPathValue(obj);
+//         }
+//         //*/
+//     }, { thresholdInTable: 0 });
+// }
+// test().catch(console.error).finally(() => process.exit());
+
+/*
+// Test passing around basic objects, and then buffers, references, etc
+class TestControllerBase {
+    async returnValue<T>(obj: T) {
+        return obj;
+    }
+}
+const TestController = SocketFunction.register(
+    "TestController-94c75a37-06db-4444-b5d0-416922f3adbc",
+    new TestControllerBase(),
+    () => ({
+        returnValue: {},
+    }),
+);
+
+async function main() {
+    let cborxInstance = new cborx.Encoder({ structuredClone: true });
+    SocketFunction.WIRE_SERIALIZER = {
+        serialize: async obj => [cborxInstance.encode(obj)],
+        deserialize: async ([buf]) => cborxInstance.decode(buf),
+    };
+    if (process.argv.includes("--server")) {
+        await SocketFunction.mount({
+            public: false,
+            port: 5051,
+            ...await getThreadKeyCert(),
+        });
+        return;
+    }
+
+
+    // Tests
+    //  - string
+    //  - number
+    //  - object with strings and number
+    //  - object with references
+    //  - array of primitives/objects
+    //  - array of primitives/objects/undefined/null
+    //  - object with buffers / typed arrays
+
+    let serverNode = TestController.nodes[getNodeId("b77f134c680b3c537.bb752907afd3de60b.querysub.com", 5051)];
+
+    async function testString() {
+        let str = "test" + Date.now();
+        let result = await serverNode.returnValue(str);
+        if (result !== str) throw new Error(`Expected ${str}, got ${result}`);
+    }
+    async function testNumber() {
+        let num = Date.now();
+        let result = await serverNode.returnValue(num);
+        if (result !== num) throw new Error(`Expected ${num}, got ${result}`);
+    }
+    async function testObject() {
+        let obj = {
+            test: "test" + Date.now(),
+            num: Date.now(),
+        };
+        let result = await serverNode.returnValue(obj);
+        if (result.test !== obj.test) throw new Error(`Expected ${obj.test}, got ${result.test}`);
+        if (result.num !== obj.num) throw new Error(`Expected ${obj.num}, got ${result.num}`);
+    }
+    async function testObjectWithReferences() {
+        let objNested = { x: 5 };
+        let obj = {
+            obj0: objNested,
+            obj1: objNested,
+            obj2: { x: 5 },
+        };
+        let result = await serverNode.returnValue(obj);
+        if (result.obj0.x !== 5) throw new Error(`Expected ${objNested.x}, got ${result.obj0.x}`);
+        if (result.obj1.x !== 5) throw new Error(`Expected ${objNested.x}, got ${result.obj1.x}`);
+        if (result.obj2.x !== 5) throw new Error(`Expected ${objNested.x}, got ${result.obj2.x}`);
+        if (result.obj0 !== result.obj1) throw new Error(`Expected ${result.obj0}, got ${result.obj1}`);
+        if (result.obj0 === result.obj2) throw new Error(`Expected ${result.obj0} !== ${result.obj2}`);
+    }
+    async function testBasicArray() {
+        let arr = ["test" + Date.now(), Date.now()];
+        let result = await serverNode.returnValue(arr);
+        if (result[0] !== arr[0]) throw new Error(`Expected ${arr[0]}, got ${result[0]}`);
+        if (result[1] !== arr[1]) throw new Error(`Expected ${arr[1]}, got ${result[1]}`);
+    }
+    async function testComplexArray() {
+        let arr = [
+            "test" + Date.now(),
+            Date.now(),
+            undefined,
+            null
+        ];
+        let result = await serverNode.returnValue(arr);
+        if (result.length !== arr.length) throw new Error(`Expected ${arr.length}, got ${result.length}`);
+        for (let i = 0; i < arr.length; i++) {
+            if (result[i] !== arr[i]) throw new Error(`Expected ${arr[i]}, got ${result[i]}`);
+        }
+    }
+
+    async function testBuffersAndTypedArrays() {
+        let obj = {
+            test0: Buffer.from(new Float64Array(Array(1024).map(x => Math.random())).buffer),
+            test1: new Float64Array([1, 2]),
+            test2: new Uint8Array([1, 2]),
+        };
+        let result = await serverNode.returnValue(obj);
+        if (!Buffer.isBuffer(result.test0)) throw new Error(`Expected buffer, got ${result.test0}`);
+        if (!result.test0.equals(obj.test0)) throw new Error(`Expected ${obj.test0}, got ${result.test0}`);
+        if (!(result.test1 instanceof Float64Array)) throw new Error(`Expected Float64Array, got ${result.test1}`);
+        if (!(result.test2 instanceof Uint8Array)) throw new Error(`Expected Uint8Array, got ${result.test2}`);
+        if (result.test1[0] !== 1) throw new Error(`Expected 1, got ${result.test1[0]}`);
+        if (result.test1[1] !== 2) throw new Error(`Expected 2, got ${result.test1[1]}`);
+        if (result.test2[0] !== 1) throw new Error(`Expected 1, got ${result.test2[0]}`);
+        if (result.test2[1] !== 2) throw new Error(`Expected 2, got ${result.test2[1]}`);
+    }
+
+    let tests: (() => Promise<void>)[] = [
+        testString,
+        testNumber,
+        testObject,
+        testObjectWithReferences,
+        testBasicArray,
+        testComplexArray,
+        testBuffersAndTypedArrays,
+    ];
+    for (let test of tests) {
+        try {
+            await test();
+        } catch (e: any) {
+            console.error(`Failed test ${test.name}, ${e.message}`);
+        }
+    }
+    process.exit();
+}
+
+logErrors(main());
+*/
+
+// async function main() {
+//     //  Ah, well... we probably won't be reading massive objects anyways. So...
+//     //      just using cborx with structuredClone is probably good enough?
+
+//     // let testFile = "C:/Users/quent/shard/backups/querysub.com-4/PathFunctionRunner/lkddfcwz.lkddd8fv.lj9juidm.8b1s28.0.__gACTgA3gEFk1spi.YMm6Zg1AEejUtImU.json";
+//     // //testFile = "C:/Users/quent/shard/backups/1-querysub.com/PathFunctionRunner/lj1z7won.lj1z5rrz.lj1z3oe0.2zy9m.0.json";
+
+//     // const testBuffer = fs.readFileSync(testFile);
+//     // let objs = await JSONLACKS.parseLines(testBuffer);
+
+//     // objs = [
+//     //     {
+//     //         test: Buffer.from(new Float64Array(Array(1024 * 1024 * 64).map(x => Math.random())).buffer)
+//     //     },
+//     //     {
+//     //         test: Buffer.from(new Float64Array(Array(1024 * 1024 * 64).map(x => Math.random())).buffer)
+//     //     },
+//     //     {
+//     //         test: Buffer.from(new Float64Array(Array(1024 * 1024 * 64).map(x => Math.random())).buffer)
+//     //     },
+//     //     {
+//     //         test: Buffer.from(new Float64Array(Array(1024 * 1024 * 64).map(x => Math.random())).buffer)
+//     //     }
+//     // ];
+
+//     // console.log(`${formatNumber(testBuffer.length)} bytes, ${formatNumber(objs.length)} objects`);
+
+//     // await measureCode(async function runWithMsgpack() {
+//     //     let binary = measureBlock(function encode() {
+//     //         return msgpack.encode(objs);
+//     //     });
+//     //     let newObjs = measureBlock(function decode() {
+//     //         return msgpack.decode(binary) as any;
+//     //     });
+//     //     console.log(`msgpack = ${formatNumber(binary.length)} bytes, count ${newObjs.length}`);
+//     // });
+
+//     // await measureCode(async function runWithCborx() {
+//     //     let binary = measureBlock(function encode() {
+//     //         return cborx.encode(objs);
+//     //     });
+//     //     let newObjs = measureBlock(function decode() {
+//     //         return cborx.decode(binary);
+//     //     });
+//     //     console.log(`cborx = ${formatNumber(binary.length)} bytes, count ${newObjs.length}`);
+//     // });
+
+//     // await measureCode(async function runWithCborxStructured() {
+//     //     let encoder = new cborx.Encoder({ structuredClone: true });
+//     //     let binary = measureBlock(function encode() {
+//     //         return encoder.encode(objs);
+//     //     });
+//     //     let newObjs = measureBlock(function decode() {
+//     //         return encoder.decode(binary);
+//     //     });
+//     //     console.log(`cborx structured = ${formatNumber(binary.length)} bytes, count ${newObjs.length}`);
+//     // });
+
+
+//     // let archives = getArchives("test");
+
+//     // await archives.set("folder/a.txt", Buffer.from("test" + Date.now()));
+//     // await archives.set("folder/b.txt", Buffer.from("test" + Date.now()));
+//     // await archives.set("folder/nested/1.txt", Buffer.from("test" + Date.now()));
+
+//     // console.log("all files", await archives.find(""));
+//     // console.log("root folders", await archives.find("", { shallow: true, type: "folders" }));
+//     // console.log("all folders", await archives.find("", { type: "folders" }));
+//     // console.log("nested files", await archives.find("folder/", { shallow: true, type: "files" }));
+//     // console.log("nested folders", await archives.find("folder/", { shallow: true, type: "folders" }));
+
+//     // process.exit();
+
+//     // await archives.set("nested/testFile.txt", Buffer.from("test" + Date.now()));
+//     // console.log("testResult", (await archives.get("nested/testFile.txt"))?.toString());
+//     // let info = await archives.getInfo("nested/testFile.txt");
+//     // console.log("info", info);
+//     // console.log("files", await archives.find("nested/"));
+//     // await archives.del("nested/testFile.txt");
+//     // console.log("files", await archives.find("nested/"));
+//     // console.log("testResult", (await archives.get("nested/testFile.txt"))?.toString());
+
+//     // SocketFunction.silent = false;
+//     // await SocketFunction.mount({
+//     //     public: false,
+//     //     port: 0,
+//     //     ...await getThreadKeyCert(),
+//     // });
+//     // watchNodeIds(nodeIds => {
+//     //     console.log(`Node ids changed at ${new Date().toISOString()}`);
+//     //     for (let id of nodeIds) {
+//     //         console.log(`  ${id}`);
+//     //     }
+//     //     console.log();
+//     // });
+// }
+
+// logErrors(main());
+
+
+
+
+//*
+
+
+// let { data, functions } = Querysub.createSchema<{
+//     ownedValues: {
+//         [key: string]: string;
+//     },
+//     values: {
+//         [index: number]: {
+//             sum: number;
+//             lastUpdated: number;
+//             lastUpdatedBy: string;
+//         }
+//     };
+// }>()({
+//     domainName: "querysub.com",
+//     functions: {
+//         setValues(indexes: number[], value?: number) {
+//             for (let index of indexes) {
+//                 let newValue = value ?? (data().values[index].sum + 1);
+//                 data().values[index] = {
+//                     sum: newValue,
+//                     lastUpdated: Querysub.now(),
+//                     lastUpdatedBy: Querysub.getCallerMachineId(),
+//                 };
+//             }
+//         },
+//         setOwnedValue(key: string, value: string) {
+//             data().ownedValues[key] = value;
+//         }
+//     },
+//     module,
+//     moduleId: "test",
+//     functionMetadata: {},
+//     permissions: {
+//         PERMISSIONS: isRegisteredUserCheck,
+//         ownedValues: {
+//             ["*"]: {
+//                 PERMISSIONS: (config) => {
+//                     return config.pathWildcards?.[0] === config.callerMachineId;
+//                 },
+//             }
+//         }
+//     }
+// });
+
+function isRegisteredUserCheck(): boolean {
+    let allowed = Querysub.getCallerIP() === "127.0.0.1";
+    if (!allowed) {
+        console.log(red(`Permission denied to IP ${JSON.stringify(Querysub.getCallerIP())}`));
+    }
+    return allowed;
+}
+
+
+
+const INDEX_EXTENT = 300 * 300;
+//const INDEX_EXTENT = 100 * 100;
+
+let getNodeHue = cache((nodeId: string) => {
+    return fastHash(nodeId) % 360;
+});
+
+
+class Page extends qreact.Component {
+
+    state = {
+        results: atomicObjectWrite([]) as string[],
+        canvas: undefined as undefined | {
+            value: HTMLCanvasElement;
+            context: CanvasRenderingContext2D;
+        },
+        writeTime: 0,
+        key: "",
+    };
+
+    onUnmount = new PromiseObj();
+    componentDidMount(): void {
+        let index = 0;
+        // NOTE: This ends up being a lot slower, due to the massive overhead of having a watcher
+        //  with so many watched paths.
+        // setInterval(() => {
+        //     functions.setValues([index++]);
+        // }, 100);
+    }
+    componentWillUnmount(): void {
+        this.onUnmount.resolve();
+    }
+    render() {
+        let canvasObj = atomic(this.state.canvas);
+        if (canvasObj) {
+            let watcher = proxyWatcher.getTriggeredWatcher();
+            watcher.pendingWatches.paths = new Set(watcher.lastWatches.paths);
+            watcher.pendingWatches.parentPaths = new Set(watcher.lastWatches.parentPaths);
+            if (watcher.pendingWatches.paths.size < 1000) {
+                for (let v = 0; v < INDEX_EXTENT; v++) {
+                    sat_data().values[v];
+                }
+            }
+
+            let changes = watcher.triggeredByChanges;
+            let indexesChanged = new Set<number>();
+            if (changes && changes.paths.size < 1000) {
+                for (let path of changes?.paths) {
+                    if (path.startsWith(".,querysub._com.,PathFunctionRunner.,saturationTest.,Data.,values.,")) {
+                        indexesChanged.add(Number(path.split(".,").at(-2)));
+                    }
+                }
+            } else {
+                for (let v = 0; v < INDEX_EXTENT; v++) {
+                    indexesChanged.add(v);
+                }
+            }
+            let { value: canvas, context } = canvasObj;
+            let imageData = context.getImageData(0, 0, canvas.width, canvas.height);
+            let imageArray = imageData.data;
+            for (let v of indexesChanged) {
+                let i = v * 4;
+                let obj = sat_data().values[v];
+                let baseSum = Number(obj);
+                let sum = Number(baseSum) % 3;
+
+                let hue = 180;
+                let saturation = (0.5 + sum / 3 * 0.5) * 100;
+                let lightness = (0.5 + sum / 3 * 0.5) * 100;
+                if (!baseSum) {
+                    lightness = 100;
+                }
+
+                let rgb = hslToRGB({ h: hue, s: saturation, l: lightness });
+
+                imageArray[i + 0] = rgb.r;
+                imageArray[i + 1] = rgb.g;
+                imageArray[i + 2] = rgb.b;
+                imageArray[i + 3] = 255;
+            }
+            context.putImageData(imageData, 0, 0);
+        }
+
+
+        return (
+            <div>
+                <div>Calls {formatStats(Benchmark.getLiveCallStats(), { noColor: true })}</div>
+                <div>Rate: {Math.round(Benchmark.getLiveRate() * 1000 * 10) / 10}/s</div>
+                <div>Saturated: {percent(Benchmark.saturatedFraction())}</div>
+                <div>Predict {Benchmark.predictCount}, received {Benchmark.receivedCount}</div>
+                <div>Open {Benchmark.overlapRate.openCount}, time waiting {Benchmark.overlapRate.finishedTime}</div>
+                <div>Machine Id {Querysub.getSelfMachineId()}</div>
+                {/* <div>
+                    <label>
+                        Key <input value={this.state.key} onChange={e => this.state.key = e.currentTarget.value} />
+                    </label>
+                    <div>
+                        Value: <input
+                            value={data().ownedValues[this.state.key]}
+                            onChange={e => functions.setOwnedValue(this.state.key, e.currentTarget.value)}
+                        />
+                    </div>
+                </div>
+                <div>
+                    Owned values keys (should be empty) {Object.keys(data().ownedValues).join(", ")}
+                </div> */}
+                <div style={{ display: "flex", gap: 10 }}>
+                    <button onClick={() => {
+                        sat_functions.setValues([0]);
+                    }}>
+                        Single write
+                    </button>
+                    <button onClick={() => {
+                        //functions.setValues(list(INDEX_EXTENT));
+                        setImmediate(async () => {
+                            let time = Date.now();
+                            let call = sat_functions.setValues(list(INDEX_EXTENT), 0);
+                            //let call = functions.setValues(list(10 * 1000), 0);
+                            await Benchmark.waitForCallToFinish(call);
+                            time = Date.now() - time;
+                            Querysub.serviceWriteDetached(() => {
+                                this.state.writeTime = time;
+                            });
+                        });
+                    }}>
+                        Value Fill
+                    </button>
+                    <div>
+                        Write time: {formatTime(this.state.writeTime)}
+                    </div>
+                </div>
+                <div>
+                    <canvas
+                        width={Math.ceil(Math.sqrt(INDEX_EXTENT))}
+                        height={Math.ceil(Math.sqrt(INDEX_EXTENT))}
+                        style={{
+                            border: "1px solid black",
+                            zoom: 800 / Math.sqrt(INDEX_EXTENT),
+                            imageRendering: "pixelated",
+                        }}
+                        ref={canvas => {
+                            if (canvas && canvas !== this.state.canvas?.value) {
+                                this.state.canvas = atomicObjectWrite({
+                                    value: canvas,
+                                    context: canvas.getContext("2d")!,
+                                });
+                            }
+                        }}
+                    />
+                </div>
+                <pre>
+                    {this.state.results.map(result => <div>{result || <>&nbsp;</>}</div>)}
+                </pre>
+            </div>
+        );
+    }
+}
+
+async function browserMain() {
+    if (isNode()) return;
+
+    watchFilesAndTriggerHotReloading();
+
+    //Querysub.debug();
+    // ClientWatcher.DEBUG_READS = true;
+    // ClientWatcher.DEBUG_WRITES = true;
+    // Querysub.DEBUG_CALLS = true;
+
+    qreact.render(qreact.createElement(Page, {}), document.getElementById("main")!);
+}
+
+async function main() {
+    if (!isNode()) return;
+    if (isDeploy()) return;
+    if (module.hotreload) return;
+    const edgePort = 5050;
+    PermissionsCheck.DEBUG = true;
+    // ClientWatcher.DEBUG_READS = true;
+    // ClientWatcher.DEBUG_WRITES = true;
+    //debugCoreMode();
+
+    //SocketFunction.silent = false;
+    Error.stackTraceLimit = 20;
+
+    await Querysub.hostServer({
+        trustedDomains: ["querysub.com"],
+        port: edgePort,
+        rootPath: __dirname + "/../../",
+        clientsideEntryPoint: "./src/test/test.tsx",
+        hotReload: true,
+    });
+}
+
+module.hotreload = isNode() && process.argv[1].includes("PathFunctionRunnerMain.ts");
+module.noserverhotreload = false;
+
+logErrors(main());
+logErrors(browserMain());
+
+//*/

package/src/zip.ts

@@ -0,0 +1,92 @@
+import { isNode } from "socket-function/src/misc";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import zlib from "zlib";
+import * as pako from "pako";
+
+import { setFlag } from "socket-function/require/compileFlags";
+import { unzipThreaded } from "./zipThreaded";
+import debugbreak from "debugbreak";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { runInSerial } from "socket-function/src/batching";
+setFlag(require, "pako", "allowclient", true);
+
+export class Zip {
+    @measureFnc
+    public static async gzip(buffer: Buffer, level?: number): Promise<Buffer> {
+        if (isNode()) {
+            return new Promise((resolve, reject) => {
+                zlib.gzip(buffer, { level }, (err: any, result: Buffer) => {
+                    if (err) reject(err);
+                    else resolve(result);
+                });
+            });
+        } else {
+            // @ts-ignore
+            return await doStream(new CompressionStream("gzip"), buffer);
+        }
+    }
+    @measureFnc
+    public static async gunzip(buffer: Buffer): Promise<Buffer> {
+        return Zip.gunzipUntracked(buffer);
+    }
+    // A base function, so we can avoid instrumentation for batch calls
+    public static async gunzipUntracked(buffer: Buffer): Promise<Buffer> {
+        if (isNode()) {
+            let test = await unzipThreaded(buffer);
+            // let test2 = await new Promise<Buffer>((resolve, reject) => {
+            //     zlib.gunzip(buffer, (err: any, result: Buffer) => {
+            //         if (err) reject(err);
+            //         else resolve(result);
+            //     });
+            // });
+            // if (test.length !== test2.length) {
+            //     debugbreak(2);
+            //     debugger;
+            // }
+            // for (let i = 0; i < test.length; i++) {
+            //     if (test[i] !== test2[i]) {
+            //         debugbreak(2);
+            //         debugger;
+            //     }
+            // }
+
+            return test;
+        } else {
+            // NOTE: pako seems to be faster, at least clientside.
+            // TIMING: 700ms vs 1200ms
+            //  - This might just be faster for small files.
+            return Buffer.from(pako.inflate(buffer));
+            // @ts-ignore
+            // return await doStream(new DecompressionStream("gzip"), buffer);
+        }
+    }
+
+    @measureFnc
+    public static async gunzipBatch(buffers: Buffer[]): Promise<Buffer[]> {
+        let time = Date.now();
+        buffers = await Promise.all(buffers.map(Zip.gunzipUntracked));
+        time = Date.now() - time;
+        let totalSize = buffers.reduce((acc, buffer) => acc + buffer.length, 0);
+        //console.log(`Gunzip ${formatNumber(totalSize)}B at ${formatNumber(totalSize / time * 1000)}B/s`);
+        return buffers;
+    }
+}
+
+async function doStream(stream: GenericTransformStream, buffer: Buffer): Promise<Buffer> {
+    let reader = stream.readable.getReader();
+    let writer = stream.writable.getWriter();
+    let writePromise = writer.write(buffer);
+    let closePromise = writer.close();
+
+    let outputBuffers: Buffer[] = [];
+    while (true) {
+        let { value, done } = await reader.read();
+        if (done) {
+            await writePromise;
+            await closePromise;
+            return Buffer.concat(outputBuffers);
+        }
+        outputBuffers.push(Buffer.from(value));
+    }
+
+}