querysub 0.2.0

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -0,0 +1,769 @@
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { PathValue, ReadLock, Time } from "../0-path-value-core/pathValueCore";
+
+import { lazy } from "socket-function/src/caching";
+import debugbreak from "debugbreak";
+import { compare, recursiveFreeze } from "socket-function/src/misc";
+import { MaybePromise } from "socket-function/src/types";
+import { StringSerialize } from "./stringSerializer";
+import { Zip } from "../zip";
+import { asBuffer, asInt32, asUint32 } from "../buffers";
+import { setFlag } from "socket-function/require/compileFlags";
+
+import cbor from "cbor-x";
+import { atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites } from "../2-proxy/PathValueProxyWatcher";
+import { formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
+setFlag(require, "cbor-x", "allowclient", true);
+
+import * as pako from "pako";
+import { delay } from "socket-function/src/batching";
+setFlag(require, "pako", "allowclient", true);
+
+const DEFAULT_BUFFER_SIZE = 1024 * 16;
+const MAX_BUFFER_SIZE = 1024 * 1024 * 64;
+const INPLACE_BUFFER_THRESHOLD = 1024 * 1024 * 16;
+
+// Often times large data is compressed at an application level, If we compress it again,
+//  it means readers will have to store the compressed buffer AND uncompressed buffer in memory,
+//  which can result in a lot of extra memory pressure. Not to mention wasted time uncompressed it.
+const MIN_COMPRESS_FACTOR = 0.75;
+
+const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
+
+const SERIALIZE_LOCK_COUNT = true;
+
+/*
+    Memory Layout
+        DataSettings
+        main data (possibly compressed)
+            PathValueStructure[]
+            ReadLock[][]
+            values[]
+            string dictionary
+*/
+
+// TODO: Add a endian version in settings, so we can switch to LE
+//      - Maybe we should benchmark LE vs BE first, but... I assume LE
+//          will be faster, as that is the native format for most CPUs.
+
+// NOTE: We could heavily optimize how we store Times
+//  - Most versions will be 0, and all will be small, so we can pack version and time
+//  - Most creatorIds are the same, so we could group by creatorId
+//  - endTime will be highly reused, so we can probably share them in some way
+//      (although, if Time is already just 64 bits, I guess we don't even need this optimization).
+
+interface Writer {
+    writeFloat64(value: number): void;
+    writeByte(value: number): void;
+    writeString(value: string): void;
+    writeBuffer(value: Buffer): void;
+}
+interface Reader {
+    readFloat64(): number;
+    readByte(): number;
+    readString(): string;
+    readBuffer(): Buffer;
+}
+
+// JSON.stringify, length prefixed
+interface DataSettings {
+    compression?: "gzip";
+    valueCount: number;
+    noLocks?: boolean;
+    stripSource?: boolean;
+    version: (
+        undefined
+        //  Split strings into multiple buffers
+        | 1
+    );
+}
+
+interface PathValueStructure {
+    path: string;
+    time: Time;
+
+    event: boolean;
+    isTransparent: boolean;
+    valid: boolean;
+    canGCValue: boolean;
+    source?: string;
+    lockCount: number;
+    updateCount: number;
+    // IMPORTANT! All of these values must be explicitly copied to PathValue
+    //  (we don't spread it), so if you add a new value here, make sure to
+    //  update places that use these values (search for canGCValue in this file).
+}
+
+
+
+class PathValueSerializer {
+    @measureFnc
+    private pathValuesWrite(writer: Writer, pathValues: PathValue[], settings: DataSettings) {
+        for (let pathValue of pathValues) {
+            writer.writeString(pathValue.path);
+            this.timeWrite(writer, pathValue.time);
+
+            let hasSource = pathValue.source && !settings.stripSource;
+            let flag0 = (
+                (pathValue.event ? 1 : 0)
+                | (pathValue.isTransparent ? 2 : 0)
+                | (pathValue.valid ? 4 : 0)
+                | (pathValue.canGCValue ? 8 : 0)
+                | (hasSource ? 16 : 0)
+                | (SERIALIZE_LOCK_COUNT ? 32 : 0)
+                | 64
+            );
+            writer.writeByte(flag0);
+            // NOTE: If we ever need flag1, set the last bit of flag0 (| 128),
+            //  and then write another flag.
+
+            if (hasSource) {
+                writer.writeString(pathValue.source!);
+            }
+
+            if (SERIALIZE_LOCK_COUNT) {
+                writer.writeFloat64(pathValue.locks.length || pathValue.lockCount || 0);
+            }
+
+            writer.writeFloat64(pathValue.updateCount || 0);
+        }
+    }
+
+    @measureFnc
+    private pathValuesRead(reader: Reader, count: number): PathValueStructure[] {
+        let pathValues: PathValueStructure[] = [];
+        for (let i = 0; i < count; i++) {
+
+            let path = reader.readString();
+            let time = this.timeRead(reader);
+
+            let flags: number[] = [];
+            while (true) {
+                let flag = reader.readByte();
+                flags.push(flag);
+                if (!(flag & 128)) break;
+            }
+            let flag0 = flags[0] || 0;
+            let event = (flag0 & 1) !== 0;
+            let isTransparent = (flag0 & 2) !== 0;
+            let valid = (flag0 & 4) !== 0;
+            let canGCValue = (flag0 & 8) !== 0;
+            let hasSource = (flag0 & 16) !== 0 ? true : undefined;
+            let hasLockCount = (flag0 & 32) !== 0;
+            let hasUpdateCount = (flag0 & 64) !== 0;
+
+            let source: string | undefined;
+            if (hasSource) {
+                source = reader.readString();
+            }
+
+            let lockCount = 0;
+            if (hasLockCount) {
+                lockCount = reader.readFloat64();
+            }
+            let updateCount = 0;
+            if (hasUpdateCount) {
+                updateCount = reader.readFloat64();
+            }
+
+            pathValues.push({
+                path,
+                time,
+
+                event,
+                isTransparent,
+                valid,
+                canGCValue,
+                source,
+                lockCount,
+                updateCount,
+            });
+        }
+        return pathValues;
+    }
+
+    private timeWrite(writer: Writer, time: Time) {
+        writer.writeFloat64(time.time);
+        writer.writeFloat64(time.version);
+        writer.writeFloat64(time.creatorId);
+    }
+    private timeRead(reader: Reader): Time {
+        // NOTE: We COULD pack time + version, as version will always be small, and time will always
+        //  use < 42 bits (until the year 2109).
+        return {
+            time: reader.readFloat64(),
+            version: reader.readFloat64(),
+            creatorId: reader.readFloat64(),
+        };
+    }
+
+
+    @measureFnc
+    private readLocksWrite(writer: Writer, readLocks: ReadLock[][]) {
+        let readLockIndexes = new Map<ReadLock[], number>();
+        for (let readLock of readLocks) {
+            if (readLockIndexes.has(readLock)) continue;
+            if (readLock.length === 0) {
+                readLockIndexes.set(readLock, -1);
+                continue;
+            }
+            readLockIndexes.set(readLock, readLockIndexes.size);
+        }
+
+        let uniqueLocks = Array.from(readLockIndexes.keys());
+        writer.writeFloat64(uniqueLocks.length);
+        for (let readLockList of uniqueLocks) {
+            writer.writeFloat64(readLockList.length);
+            for (let readLock of readLockList) {
+                writer.writeString(readLock.path);
+                this.timeWrite(writer, readLock.startTime);
+                this.timeWrite(writer, readLock.endTime);
+                writer.writeByte(readLock.readIsTranparent ? 1 : 0);
+            }
+        }
+
+        let indexes = readLocks.map(x => readLockIndexes.get(x)!);
+        writer.writeBuffer(asBuffer(new Int32Array(indexes)));
+    }
+    @measureFnc
+    private readLocksRead(reader: Reader): ReadLock[][] {
+        let uniqueLockCount = reader.readFloat64();
+        let uniqueLocks: ReadLock[][] = [];
+        for (let i = 0; i < uniqueLockCount; i++) {
+            let lockCount = reader.readFloat64();
+            let locks: ReadLock[] = [];
+            for (let j = 0; j < lockCount; j++) {
+                locks.push({
+                    path: reader.readString(),
+                    startTime: this.timeRead(reader),
+                    endTime: this.timeRead(reader),
+                    readIsTranparent: reader.readByte() === 1,
+                });
+            }
+            uniqueLocks.push(locks);
+        }
+
+        let indexesBuf = reader.readBuffer();
+        let indexes: number[] = [];
+        for (let i = 0; i < indexesBuf.length; i += 4) {
+            indexes.push(indexesBuf.readInt32LE(i));
+        }
+        let emptyLocks: ReadLock[] = [];
+        let readLocks = Array.from(indexes).map(index => {
+            if (index === -1) return emptyLocks;
+            if (index < 0 || index >= uniqueLocks.length) {
+                debugbreak(2);
+                debugger;
+                throw new Error(`Invalid read lock index ${index}, expected 0 <= index < ${uniqueLocks.length}`);
+            }
+            return uniqueLocks[index];
+        });
+        return readLocks;
+    }
+
+
+    private async valuesWrite(writer: Writer, valuesGroups: PathValue[][]) {
+        let totalLength = valuesGroups.reduce((total, x) => total + x.length, 0);
+        let types = Buffer.alloc(totalLength);
+        let encodedValues: (Buffer | number)[] = [];
+
+        let i = 0;
+        for (let values of valuesGroups) {
+            measureBlock(() => {
+                // NOTE: Writing one value at a time is about 1.6X slower. BUT, it allows us to very efficient decode
+                //  values, which... probably saves time overall.
+                //  - ALSO, wire time is probably the slowest part anyways
+                for (let k = 0; k < values.length; k++) {
+                    let pathValue = values[k];
+                    let value = pathValue.value;
+                    if (pathValue.isValueLazy) {
+                        // If it's already encoded that's great, we can just reuse that buffer in our new encoding
+                        types[i] = 6;
+                        // Call getBuffer, in case we lost the lazyValue somehow. This is fine even
+                        //  if it is a number (although a number should NEVER have isValueLazy set),
+                        //  because numbers can be encoded as Buffers (it is just inefficient).
+                        encodedValues.push(this.getBuffer(pathValue));
+                    } else if (value === undefined) {
+                        types[i] = 1;
+                    } else if (value === null) {
+                        types[i] = 2;
+                    } else if (value === false) {
+                        types[i] = 3;
+                    } else if (value === true) {
+                        types[i] = 4;
+                    } else if (typeof value === "number") {
+                        types[i] = 5;
+                        encodedValues.push(value);
+                    } else {
+                        types[i] = 6;
+                        encodedValues.push(cborEncoder().encode(value));
+                    }
+                    i++;
+                }
+            }, "valuesWriteLoop");
+            await delay("paintLoop");
+        }
+
+        measureBlock(function valuesWriteTypesBuffer() {
+            writer.writeBuffer(types);
+        });
+        // Break encodeValues into groups of 100, so we can check the time and delay
+        //      if we are taking too long, but not check it EVERY loop, as that would
+        //      be too slow.
+        let valueGroups: (Buffer | number)[][] = [];
+        const VALUE_GROUP_SIZE = 1000;
+        for (let i = 0; i < encodedValues.length; i += VALUE_GROUP_SIZE) {
+            valueGroups.push(encodedValues.slice(i, i + VALUE_GROUP_SIZE));
+        }
+        let prevTime = Date.now();
+        for (let encodedValues of valueGroups) {
+            measureBlock(function valuesWriteTypes() {
+                for (let value of encodedValues) {
+                    if (typeof value === "number") {
+                        writer.writeFloat64(value);
+                    } else {
+                        writer.writeBuffer(value);
+                    }
+                }
+            });
+            let now = Date.now();
+            if (now - prevTime > 10) {
+                await delay("paintLoop");
+                prevTime = now;
+            }
+        }
+    }
+    @measureFnc
+    private valuesRead(reader: Reader, count: number, lazy: boolean): {
+        values: unknown[];
+        valuesAreLazy: boolean[];
+    } {
+        let values: unknown[] = [];
+        let valuesAreLazy: boolean[] = [];
+
+        let types = reader.readBuffer();
+        for (let i = 0; i < count; i++) {
+            let type = types[i];
+            let isLazy = false;
+            if (type === 1) {
+                values.push(undefined);
+            } else if (type === 2) {
+                values.push(null);
+            } else if (type === 3) {
+                values.push(false);
+            } else if (type === 4) {
+                values.push(true);
+            } else if (type === 5) {
+                values.push(reader.readFloat64());
+            } else {
+                let buf = reader.readBuffer();
+                if (lazy) {
+                    values.push(this.addLazyValue(buf));
+                    isLazy = true;
+                } else {
+                    // Freeze the read values, as we really never want them to modified
+                    //  (We don't freeze PathValue as applications don't directly access them, and the freeze overhead isn't free.
+                    //      But applications DO access values, and if they change them it breaks so many things, so it is worth the
+                    //      cost to freeze them).
+                    // NOTE: This makes the code about `10% slower, so... it really not that slow (although in our test we had
+                    //      a lot of other overhead, so... maybe it will be a lot slower in certain cases?)
+                    values.push(recursiveFreeze(cbor.decode(buf)));
+                }
+            }
+            valuesAreLazy.push(isLazy);
+        }
+
+        return { values, valuesAreLazy };
+        // let valuesBuffer = reader.readBuffer();
+        // return cbor.decode(valuesBuffer);
+    }
+
+    @measureFnc
+    public async serialize(values: PathValue[], config?: {
+        noLocks?: boolean;
+        // TIMING: Compression is about 2X slower (both compression and decompression), but reduces the size by about 2X
+        compress?: boolean;
+        singleBuffer?: boolean;
+        stripSource?: boolean;
+    }): Promise<Buffer[]> {
+        const version = 1;
+        let settings: DataSettings = {
+            valueCount: values.length,
+            noLocks: config?.noLocks,
+            compression: config?.compress ? "gzip" : undefined,
+            stripSource: config?.stripSource,
+            version,
+        };
+
+        let totalSize = 0;
+        let currentBufferDefaultSize = DEFAULT_BUFFER_SIZE;
+        let outputBuffers: Buffer[] = [];
+        let currentBuffer = Buffer.alloc(currentBufferDefaultSize);
+        let currentBufferPos = 0;
+        function ensureBytes(count: number) {
+            if (currentBufferPos + count < currentBuffer.length) return;
+            if (currentBufferPos !== currentBuffer.length) {
+                currentBuffer = currentBuffer.slice(0, currentBufferPos);
+            }
+            outputBuffers.push(currentBuffer);
+            totalSize += currentBufferPos;
+            currentBufferDefaultSize = Math.min(MAX_BUFFER_SIZE, Math.max(currentBufferDefaultSize, totalSize * 0.25));
+            currentBuffer = Buffer.alloc(Math.max(currentBufferDefaultSize, count));
+            currentBufferPos = 0;
+        }
+
+        let stringDictionary = new Map<string, number>();
+        let writer: Writer = {
+            writeFloat64(value) {
+                ensureBytes(8);
+                if (version >= 1) {
+                    currentBuffer.writeDoubleLE(value, currentBufferPos);
+                } else {
+                    currentBuffer.writeDoubleBE(value, currentBufferPos);
+                }
+                currentBufferPos += 8;
+            },
+            writeByte(value) {
+                ensureBytes(1);
+                currentBuffer[currentBufferPos] = value;
+                currentBufferPos++;
+            },
+            writeString(value) {
+                let index = stringDictionary.get(value);
+                if (index === undefined) {
+                    index = stringDictionary.size;
+                    stringDictionary.set(value, index);
+                }
+                writer.writeFloat64(index);
+            },
+            writeBuffer(value) {
+                // NOTE: I think the inplace optimization is actually slower? We needed to add a length prefix
+                //      buffer anyways, and currentBufferDefaultSize would drift upwards, resulting
+                //      in allocating 1GB, slicing the first 8 bytes, adding the large in place buffer,
+                //      allocating 1GB, etc.
+                // Special case, very large buffers. Don't copy them, just add the whole buffer
+                // if (value.length > INPLACE_BUFFER_THRESHOLD || value.length > currentBuffer.length) {
+                //     writer.writeFloat64(value.length);
+                //     if (currentBufferPos > 0) {
+                //         outputBuffers.push(currentBuffer.slice(0, currentBufferPos));
+                //         currentBuffer = Buffer.alloc(currentBufferDefaultSize);
+                //         currentBufferPos = 0;
+                //     }
+                //     outputBuffers.push(value);
+                //     return;
+                // }
+                writer.writeFloat64(value.length);
+                ensureBytes(value.length);
+                value.copy(currentBuffer, currentBufferPos);
+                currentBufferPos += value.length;
+            },
+        };
+
+        outputBuffers.push(Buffer.from(JSON.stringify(settings)));
+
+        // Break values up into groups of 10K, so we don't lock up the main thread
+        let valueGroups: PathValue[][] = [];
+        const VALUE_GROUP_SIZE = 10_000;
+        for (let i = 0; i < values.length; i += VALUE_GROUP_SIZE) {
+            valueGroups.push(values.slice(i, i + VALUE_GROUP_SIZE));
+        }
+
+        for (let values of valueGroups) {
+            this.pathValuesWrite(writer, values, settings);
+            await delay("afterPaint");
+        }
+        if (!settings.noLocks) {
+            // TODO: Support breaking locks up
+            this.readLocksWrite(writer, values.map(x => x.locks));
+        }
+        await this.valuesWrite(writer, valueGroups);
+
+        if (currentBufferPos > 0) {
+            outputBuffers.push(currentBuffer.slice(0, currentBufferPos));
+        }
+
+        // NOTE: The dictionary is ordered, so we don't need to sort by the values or anything
+        let strings = Array.from(stringDictionary.keys());
+        // Assume each character takes up 2 bytes. It will usually take just 1, but... smaller is fine here.
+        const MAX_STRING_LENGTH = Math.ceil(MAX_BUFFER_SIZE / 2);
+        let stringParts: string[][] = [];
+        let curCount = 0;
+        let curStrings: string[] = [];
+        stringParts.push(curStrings);
+        for (let str of strings) {
+            if (curCount + str.length > MAX_STRING_LENGTH) {
+                curStrings = [];
+                stringParts.push(curStrings);
+                curCount = 0;
+            }
+            curStrings.push(str);
+            curCount += str.length;
+        }
+
+        let stringBuffers: Buffer[] = [];
+        stringBuffers.push(asBuffer(new Float64Array([stringParts.length])));
+        for (let strings of stringParts) {
+            stringBuffers.push(StringSerialize.serializeStrings(strings));
+            await delay("paintLoop");
+        }
+        // Add reversed, so they can be read in backwards order
+        stringBuffers.reverse();
+        outputBuffers.push(...stringBuffers);
+
+        if (settings.compression === "gzip") {
+            // NOTE: Due to how the LZ77 window works merging buffers probably won't reduce the size by that much.
+            let compressedBuffers = await Promise.all(outputBuffers.slice(1).map(x => Zip.gzip(x, 1)));
+            let compressedOutputBuffers = [outputBuffers[0], ...compressedBuffers];
+
+            // If the compress factor is less than a threshold, use the uncompressed buffers
+            let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
+            let compressedSize = compressedOutputBuffers.reduce((total, x) => total + x.length, 0);
+            if (compressedSize / uncompressedSize < MIN_COMPRESS_FACTOR) {
+                outputBuffers = compressedOutputBuffers;
+            } else {
+                settings.compression = undefined;
+                outputBuffers[0] = Buffer.from(JSON.stringify(settings));
+            }
+        }
+
+        if (config?.singleBuffer) {
+            let sizes = Buffer.from(new Float64Array([outputBuffers.length, ...outputBuffers.map(x => x.length)]).buffer);
+            outputBuffers.unshift(sizes);
+            outputBuffers = [Buffer.concat(outputBuffers)];
+        }
+
+        return outputBuffers;
+    }
+    @measureFnc
+    public async deserialize(buffers: Buffer[], config?: {
+        // NOTE: The default is to lazily deserialize, requiring calling getPathValue to get the value.
+        fullDeserialize?: boolean;
+        singleBuffer?: boolean;
+        // All strings become empty. This can make deserialization over 2X faster.
+        skipStrings?: boolean;
+        // All values become undefined. If also skipping strings OR if you have large values,
+        //  this can make deserialization significantly faster.
+        skipValues?: boolean;
+    }): Promise<PathValue[]> {
+        buffers = buffers.slice();
+
+        if (config?.singleBuffer) {
+            const buffer = buffers[0];
+            let bufferCount = buffer.readDoubleLE(0);
+            let bufferLengths: number[] = [];
+            for (let i = 1; i <= bufferCount; i++) {
+                let length = buffer.readDoubleLE(i * 8);
+                if (length < 0 || length > buffer.byteLength) {
+                    throw new Error(`Corrupted data file. Invalid buffer length ${length}, expected 0 <= length <= ${buffer.byteLength}`);
+                }
+                bufferLengths.push(length);
+            }
+
+            let newBuffers: Buffer[] = [];
+            let offset = 8 + bufferCount * 8;
+            for (let length of bufferLengths) {
+                let end = offset + length;
+                if (end > buffer.byteLength) {
+                    throw new Error(`Corrupted data file. Invalid buffer length ${length}, expected 0 <= length <= ${buffer.byteLength}`);
+                }
+                newBuffers.push(buffer.slice(offset, offset + length));
+                offset += length;
+            }
+            buffers = newBuffers;
+        }
+
+        let settings = JSON.parse(buffers.shift()!.toString("utf8")) as DataSettings;
+        let version = settings.version || 0;
+        if (settings.compression === "gzip") {
+            buffers = await Zip.gunzipBatch(buffers);
+        }
+
+        function getLastBuffer(): Buffer {
+            let last = buffers.pop();
+            if (!last) {
+                throw new Error(`Expected another buffer, but there are no more left.`);
+            }
+            return last;
+        }
+
+        let strings: string[] | undefined;
+        let stringBuffers: Buffer[] = [];
+        // Last buffers are the strings
+        if (version === 0) {
+            stringBuffers.push(getLastBuffer());
+        } else if (version >= 1) {
+            let countBuffer = getLastBuffer();
+            let count = countBuffer.readDoubleLE(0);
+            for (let i = 0; i < count; i++) {
+                stringBuffers.push(getLastBuffer());
+            }
+        }
+        if (!config?.skipStrings) {
+            let stringArrays: string[][] = [];
+            for (let stringBuffer of stringBuffers) {
+                let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
+                while (true) {
+                    let nextStrings = obj.getNextStrings();
+                    if (!nextStrings) {
+                        break;
+                    }
+                    stringArrays.push(nextStrings);
+                    if (stringArrays.length > 1) {
+                        await delay("paintLoop");
+                    }
+                }
+                await delay("paintLoop");
+            }
+            strings = stringArrays.flat();
+        }
+        let bufferIndex = 0;
+        let bufferPos = 0;
+        function ensureBytes(count: number) {
+            if (bufferPos + count <= buffers[bufferIndex].length) return;
+            if (bufferPos !== buffers[bufferIndex].length) {
+                throw new Error(`Reading from two buffers at once. This is unexpected.`);
+            }
+            bufferIndex++;
+            bufferPos = 0;
+        }
+        let reader: Reader = {
+            readFloat64() {
+                ensureBytes(8);
+                let result = version >= 1 ? buffers[bufferIndex].readDoubleLE(bufferPos) : buffers[bufferIndex].readDoubleBE(bufferPos);
+                bufferPos += 8;
+                return result;
+            },
+            readByte() {
+                ensureBytes(1);
+                let result = buffers[bufferIndex][bufferPos];
+                bufferPos++;
+                return result;
+            },
+            readString() {
+                ensureBytes(8);
+                let index = reader.readFloat64();
+                if (!strings) {
+                    return "";
+                }
+                if (index < 0 || index >= strings.length) {
+                    console.log(bufferIndex);
+                    throw new Error(`Invalid string index ${index}, expected 0 <= index < ${strings.length}`);
+                }
+                return strings[index];
+            },
+            readBuffer() {
+                let length = reader.readFloat64();
+                ensureBytes(length);
+                let result = buffers[bufferIndex].slice(bufferPos, bufferPos + length);
+                bufferPos += length;
+                return result;
+            },
+        };
+
+
+        let partialValues = this.pathValuesRead(reader, settings.valueCount);
+        let readLocks = settings.noLocks ? Array(settings.valueCount).fill([]) : this.readLocksRead(reader);
+
+        let values: unknown[] = [];
+        let valuesAreLazy: boolean[] = [];
+        if (!config?.skipValues) {
+            let valuesReadObj = this.valuesRead(reader, settings.valueCount, !config?.fullDeserialize);
+            values = valuesReadObj.values;
+            valuesAreLazy = valuesReadObj.valuesAreLazy;
+        }
+
+        let pathValues: PathValue[] = [];
+        for (let i = 0; i < settings.valueCount; i++) {
+            let partialValue = partialValues[i];
+            pathValues.push({
+                path: partialValue.path,
+                time: partialValue.time,
+
+                locks: readLocks[i],
+                value: values[i],
+                isValueLazy: valuesAreLazy[i] || false,
+
+                canGCValue: partialValue.canGCValue,
+                event: partialValue.event,
+                isTransparent: partialValue.isTransparent,
+                valid: partialValue.valid,
+                source: partialValue.source,
+                lockCount: partialValue.lockCount,
+                updateCount: partialValue.updateCount,
+            });
+        }
+
+        return pathValues;
+    }
+
+    lazyValues = new WeakMap<{}, Buffer>();
+    addLazyValue(buf: Buffer) {
+        // NOTE: Using an object makes garbage collection MUCH easier, basically eliminating leaks (and...
+        //  it should be fairly fast? We create a Buffer anyways... which is probably a lot slower than
+        //  creating an object).
+        let lazyInstance = { call_PathValueSerializer_getPathValue_toGetThisValue: true };
+        this.lazyValues.set(lazyInstance, buf);
+        return lazyInstance;
+    }
+
+    public getPathValue(pathValue: PathValue | undefined, noMutate?: "noMutate"): unknown {
+        if (!pathValue) return undefined;
+        if (pathValue.isValueLazy) {
+            // NOTE: If this throws, it likely means you used atomicObjectWrite.
+            //  Use atomicObjectWriteNoFreeze or doAtomicWrites instead.
+            if (!noMutate) {
+                try {
+                    pathValue.isValueLazy = false;
+                } catch { }
+            }
+            let buffer = this.lazyValues.get(pathValue.value as {});
+            if (!buffer) {
+                console.error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
+                return pathValue.value;
+            }
+            let newValue = recursiveFreeze(cbor.decode(buffer));
+            if (!noMutate) {
+                try {
+                    pathValue.value = newValue;
+                } catch { }
+            }
+            return newValue;
+        }
+        return pathValue.value;
+    }
+    public getPathValueOrBuffer(pathValue: PathValue | undefined): unknown | Buffer {
+        if (!pathValue) return undefined;
+        if (pathValue.isValueLazy) {
+            let buffer = this.lazyValues.get(pathValue.value as {});
+            if (!buffer) {
+                // NOTE: Did you pass a raw PathValue and then try to use PathValueSerializer with it?
+                //  - Instead you should pass a buffer serialized with pathValueSerializer.serialize and
+                //      deserialized with pathValueSerializer.deserialize.
+                console.error(`Expected lazy value to have a buffer, but it didn't. Lazy ref has a typeof ${typeof pathValue.value} (${String(pathValue.value)})`);
+                return pathValue.value;
+            }
+            return buffer;
+        }
+        return pathValue.value;
+    }
+
+    private getBuffer(pathValue: PathValue): Buffer {
+        let buffer = pathValue.isValueLazy && this.lazyValues.get(pathValue.value as {});
+        if (buffer) return buffer;
+        return cborEncoder().encode(pathValue.value);
+    }
+    public compareValuePaths(a: PathValue | undefined, b: PathValue | undefined) {
+        if (!a || !b) {
+            let aValue = this.getPathValue(a);
+            let bValue = this.getPathValue(b);
+            return compare(aValue, bValue);
+        }
+        let bufferA = this.getBuffer(a);
+        let bufferB = this.getBuffer(b);
+        return bufferA.compare(bufferB);
+    }
+}
+
+
+export const pathValueSerializer = new PathValueSerializer();
+(global as any).pathValueSerializer = pathValueSerializer;