querysub 0.430.0 → 0.431.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.430.0",
+  "version": "0.431.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -80,7 +80,11 @@ interface DataSettings {
         | 1
         //  Add transactionPaths support
         | 2
+        //  Separate buffers for each data type with bufferMap
+        | 3
     );
+    // Parallel map with buffers defining their type (for version 3+)
+    bufferMap?: ("pathValues" | "readLocks" | "transactionPaths" | "values" | "strings")[];
 }
 
 interface PathValueStructure {
@@ -454,14 +458,16 @@ class PathValueSerializer {
         compress?: boolean;
         singleBuffer?: boolean;
         stripSource?: boolean;
+        noTransactionPaths?: boolean;
     }): Promise<Buffer[]> {
-        const version = 2;
+        const version = 3;
         let settings: DataSettings = {
             valueCount: values.length,
             noLocks: config?.noLocks,
             compression: config?.compress ? "lz4" : undefined,
             stripSource: config?.stripSource,
             version,
+            bufferMap: [],
         };
 
         let totalSize = 0;
@@ -469,18 +475,37 @@ class PathValueSerializer {
         let outputBuffers: Buffer[] = [];
         let currentBuffer = Buffer.alloc(currentBufferDefaultSize);
         let currentBufferPos = 0;
+        let currentBufferType: "pathValues" | "readLocks" | "transactionPaths" | "values" | undefined;
+
         function ensureBytes(count: number) {
             if (currentBufferPos + count < currentBuffer.length) return;
             if (currentBufferPos !== currentBuffer.length) {
                 currentBuffer = currentBuffer.slice(0, currentBufferPos);
             }
             outputBuffers.push(currentBuffer);
+            if (currentBufferType) {
+                settings.bufferMap!.push(currentBufferType);
+            }
             totalSize += currentBufferPos;
             currentBufferDefaultSize = Math.min(MAX_BUFFER_SIZE, Math.max(currentBufferDefaultSize, totalSize * 0.25));
             currentBuffer = Buffer.alloc(Math.max(currentBufferDefaultSize, count));
             currentBufferPos = 0;
         }
 
+        function startNewBuffer(bufferType: "pathValues" | "readLocks" | "transactionPaths" | "values") {
+            if (currentBufferPos > 0) {
+                outputBuffers.push(currentBuffer.slice(0, currentBufferPos));
+                if (currentBufferType) {
+                    settings.bufferMap!.push(currentBufferType);
+                }
+            }
+            currentBufferType = bufferType;
+            totalSize += currentBufferPos;
+            currentBufferDefaultSize = Math.min(MAX_BUFFER_SIZE, Math.max(currentBufferDefaultSize, totalSize * 0.25));
+            currentBuffer = Buffer.alloc(currentBufferDefaultSize);
+            currentBufferPos = 0;
+        }
+
         let stringDictionary = new Map<string, number>();
         let writer: Writer = {
             writeFloat64(value) {
@@ -506,21 +531,6 @@ class PathValueSerializer {
                 writer.writeFloat64(index);
             },
             writeBuffer(value) {
-                // NOTE: I think the inplace optimization is actually slower? We needed to add a length prefix
-                //      buffer anyways, and currentBufferDefaultSize would drift upwards, resulting
-                //      in allocating 1GB, slicing the first 8 bytes, adding the large in place buffer,
-                //      allocating 1GB, etc.
-                // Special case, very large buffers. Don't copy them, just add the whole buffer
-                // if (value.length > INPLACE_BUFFER_THRESHOLD || value.length > currentBuffer.length) {
-                //     writer.writeFloat64(value.length);
-                //     if (currentBufferPos > 0) {
-                //         outputBuffers.push(currentBuffer.slice(0, currentBufferPos));
-                //         currentBuffer = Buffer.alloc(currentBufferDefaultSize);
-                //         currentBufferPos = 0;
-                //     }
-                //     outputBuffers.push(value);
-                //     return;
-                // }
                 writer.writeFloat64(value.length);
                 ensureBytes(value.length);
                 value.copy(currentBuffer, currentBufferPos);
@@ -528,8 +538,6 @@ class PathValueSerializer {
             },
         };
 
-        outputBuffers.push(Buffer.from(JSON.stringify(settings)));
-
         // Break values up into groups of 10K, so we don't lock up the main thread
         let valueGroups: PathValue[][] = [];
         const VALUE_GROUP_SIZE = 10_000;
@@ -537,19 +545,30 @@ class PathValueSerializer {
             valueGroups.push(values.slice(i, i + VALUE_GROUP_SIZE));
         }
 
+        startNewBuffer("pathValues");
         for (let values of valueGroups) {
             this.pathValuesWrite(writer, values, settings);
             await delay("afterPaint");
         }
+
         if (!settings.noLocks) {
-            // TODO: Support breaking locks up
+            startNewBuffer("readLocks");
             this.readLocksWrite(writer, values.map(x => x.locks));
         }
-        this.transactionPathsWrite(writer, values.map(x => x.transactionPaths));
+
+        if (!config?.noTransactionPaths) {
+            startNewBuffer("transactionPaths");
+            this.transactionPathsWrite(writer, values.map(x => x.transactionPaths));
+        }
+
+        startNewBuffer("values");
         await this.valuesWrite(writer, valueGroups);
 
         if (currentBufferPos > 0) {
             outputBuffers.push(currentBuffer.slice(0, currentBufferPos));
+            if (currentBufferType) {
+                settings.bufferMap!.push(currentBufferType);
+            }
         }
 
         // NOTE: The dictionary is ordered, so we don't need to sort by the values or anything
@@ -570,45 +589,41 @@ class PathValueSerializer {
             curCount += str.length;
         }
 
-        let stringBuffers: Buffer[] = [];
-        stringBuffers.push(asBuffer(new Float64Array([stringParts.length])));
         for (let strings of stringParts) {
-            stringBuffers.push(StringSerialize.serializeStrings(strings));
+            let stringBuffer = StringSerialize.serializeStrings(strings);
+            settings.bufferMap!.push("strings");
+            outputBuffers.push(stringBuffer);
             await delay("paintLoop");
         }
-        // Add reversed, so they can be read in backwards order
-        stringBuffers.reverse();
-        outputBuffers.push(...stringBuffers);
 
         if (settings.compression === "lz4") {
-            let compressedBuffers = await unblockLoop(outputBuffers.slice(1), x => LZ4.compress(x));
-            let compressedOutputBuffers = [outputBuffers[0], ...compressedBuffers];
+            let compressedBuffers = await unblockLoop(outputBuffers, x => LZ4.compress(x));
 
             // If the compress factor is less than a threshold, use the uncompressed buffers
             let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
-            let compressedSize = compressedOutputBuffers.reduce((total, x) => total + x.length, 0);
+            let compressedSize = compressedBuffers.reduce((total, x) => total + x.length, 0);
             if (compressedSize / uncompressedSize < MIN_COMPRESS_FACTOR) {
-                outputBuffers = compressedOutputBuffers;
+                outputBuffers = compressedBuffers;
             } else {
                 settings.compression = undefined;
-                outputBuffers[0] = Buffer.from(JSON.stringify(settings));
             }
         } else if (settings.compression === "gzip") {
             // NOTE: Due to how the LZ77 window works merging buffers probably won't reduce the size by that much.
-            let compressedBuffers = await Promise.all(outputBuffers.slice(1).map(x => Zip.gzip(x, 1)));
-            let compressedOutputBuffers = [outputBuffers[0], ...compressedBuffers];
+            let compressedBuffers = await Promise.all(outputBuffers.map(x => Zip.gzip(x, 1)));
 
             // If the compress factor is less than a threshold, use the uncompressed buffers
             let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
-            let compressedSize = compressedOutputBuffers.reduce((total, x) => total + x.length, 0);
+            let compressedSize = compressedBuffers.reduce((total, x) => total + x.length, 0);
             if (compressedSize / uncompressedSize < MIN_COMPRESS_FACTOR) {
-                outputBuffers = compressedOutputBuffers;
+                outputBuffers = compressedBuffers;
             } else {
                 settings.compression = undefined;
-                outputBuffers[0] = Buffer.from(JSON.stringify(settings));
             }
         }
 
+        // Add settings buffer at the beginning
+        outputBuffers.unshift(Buffer.from(JSON.stringify(settings)));
+
         if (config?.singleBuffer) {
             let sizes = Buffer.from(new Float64Array([outputBuffers.length, ...outputBuffers.map(x => x.length)]).buffer);
             outputBuffers.unshift(sizes);
@@ -665,6 +680,23 @@ class PathValueSerializer {
             buffers = await Zip.gunzipBatch(buffers);
         }
 
+        if (version >= 3) {
+            return this.deserializeV3(buffers, settings, config);
+        } else {
+            return this.deserializeV0toV2(buffers, settings, config);
+        }
+    }
+
+    /** @deprecated We shouldn't need this code. If we ever have to modify the file again, just delete this function entirely. As soon as we rerun a single GC, all the code gets updated to the latest version, So, even as of writing this, we don't have anything that uses the old versions. Except for old snapshots, which we'll never use. */
+    @measureFnc
+    private async deserializeV0toV2(buffers: Buffer[], settings: DataSettings, config?: {
+        fullDeserialize?: boolean;
+        skipStrings?: boolean;
+        skipValues?: boolean;
+        skipTransactionPaths?: boolean;
+    }): Promise<PathValue[]> {
+        let version = settings.version || 0;
+
         function getLastBuffer(): Buffer {
             let last = buffers.pop();
             if (!last) {
@@ -754,8 +786,6 @@ class PathValueSerializer {
         let readLocks = settings.noLocks ? Array(settings.valueCount).fill([]) : this.readLocksRead(reader);
 
         let transactionPaths: (string[] | undefined)[] = [];
-        // Skip transactions if they at all seem like they want to skip anything, as transactions are not very important. And how are you going to use transaction paths if you don't have values or if you don't have strings? 
-        // - Oh, but I think skip values just makes the values lazy. So that's normal. We still want transactions even if the values are lazy. 
         if (version > 1 && !config?.skipStrings && !config?.skipTransactionPaths) {
             transactionPaths = this.transactionPathsRead(reader);
         }
@@ -793,6 +823,167 @@ class PathValueSerializer {
         return pathValues;
     }
 
+    @measureFnc
+    private async deserializeV3(buffers: Buffer[], settings: DataSettings, config?: {
+        fullDeserialize?: boolean;
+        skipStrings?: boolean;
+        skipValues?: boolean;
+        skipTransactionPaths?: boolean;
+    }): Promise<PathValue[]> {
+        let version = settings.version || 0;
+        let bufferMap = settings.bufferMap!;
+
+        // Find string buffers and deserialize them
+        let strings: string[] | undefined;
+        if (!config?.skipStrings) {
+            let stringBufferIndexes: number[] = [];
+            for (let i = 0; i < bufferMap.length; i++) {
+                if (bufferMap[i] === "strings") {
+                    stringBufferIndexes.push(i);
+                }
+            }
+
+            let stringArrays: string[][] = [];
+            for (let bufferIndex of stringBufferIndexes) {
+                let stringBuffer = buffers[bufferIndex];
+                let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
+                while (true) {
+                    let nextStrings = obj.getNextStrings();
+                    if (!nextStrings) {
+                        break;
+                    }
+                    stringArrays.push(nextStrings);
+                    if (stringArrays.length > 1) {
+                        await delay("paintLoop");
+                    }
+                }
+                if (stringBufferIndexes.length > 1) {
+                    await delay("paintLoop");
+                }
+            }
+            strings = stringArrays.flat();
+        }
+
+        // Create a generic reader that can read from specific buffer types
+        function createReaderForBufferType(bufferType: "pathValues" | "readLocks" | "transactionPaths" | "values"): Reader {
+            let targetBufferIndexes: number[] = [];
+            for (let i = 0; i < bufferMap.length; i++) {
+                if (bufferMap[i] === bufferType) {
+                    targetBufferIndexes.push(i);
+                }
+            }
+
+            let currentBufferIndexInList = 0;
+            let bufferPos = 0;
+
+            function ensureBytes(count: number) {
+                if (currentBufferIndexInList >= targetBufferIndexes.length) {
+                    throw new Error(`No more buffers available for buffer type ${bufferType}`);
+                }
+                let currentBufferIndex = targetBufferIndexes[currentBufferIndexInList];
+                if (bufferPos + count <= buffers[currentBufferIndex].length) return;
+                if (bufferPos !== buffers[currentBufferIndex].length) {
+                    throw new Error(`Reading from two buffers at once. This is unexpected.`);
+                }
+                currentBufferIndexInList++;
+                bufferPos = 0;
+            }
+
+            return {
+                readFloat64() {
+                    ensureBytes(8);
+                    let currentBufferIndex = targetBufferIndexes[currentBufferIndexInList];
+                    let result = buffers[currentBufferIndex].readDoubleLE(bufferPos);
+                    bufferPos += 8;
+                    return result;
+                },
+                readByte() {
+                    ensureBytes(1);
+                    let currentBufferIndex = targetBufferIndexes[currentBufferIndexInList];
+                    let result = buffers[currentBufferIndex][bufferPos];
+                    bufferPos++;
+                    return result;
+                },
+                readString() {
+                    let index = this.readFloat64();
+                    if (!strings) {
+                        return "";
+                    }
+                    if (index < 0 || index >= strings.length) {
+                        throw new Error(`Invalid string index ${index}, expected 0 <= index < ${strings.length}`);
+                    }
+                    return strings[index];
+                },
+                readBuffer() {
+                    let length = this.readFloat64();
+                    ensureBytes(length);
+                    let currentBufferIndex = targetBufferIndexes[currentBufferIndexInList];
+                    let result = buffers[currentBufferIndex].slice(bufferPos, bufferPos + length);
+                    bufferPos += length;
+                    return result;
+                },
+            };
+        }
+
+        let partialValues: PathValueStructure[] = [];
+        let readLocks: ReadLock[][] = [];
+        let transactionPaths: (string[] | undefined)[] = [];
+        let values: unknown[] = [];
+        let valuesAreLazy: boolean[] = [];
+
+        // Read pathValues
+        if (bufferMap.includes("pathValues")) {
+            let reader = createReaderForBufferType("pathValues");
+            partialValues = this.pathValuesRead(reader, settings.valueCount);
+        }
+
+        // Read readLocks
+        if (!settings.noLocks && bufferMap.includes("readLocks")) {
+            let reader = createReaderForBufferType("readLocks");
+            readLocks = this.readLocksRead(reader);
+        } else {
+            readLocks = Array(settings.valueCount).fill([]);
+        }
+
+        // Read transactionPaths
+        if (!config?.skipStrings && !config?.skipTransactionPaths && bufferMap.includes("transactionPaths")) {
+            let reader = createReaderForBufferType("transactionPaths");
+            transactionPaths = this.transactionPathsRead(reader);
+        }
+
+        // Read values
+        if (!config?.skipValues && bufferMap.includes("values")) {
+            let reader = createReaderForBufferType("values");
+            let valuesReadObj = this.valuesRead(reader, settings.valueCount, !config?.fullDeserialize);
+            values = valuesReadObj.values;
+            valuesAreLazy = valuesReadObj.valuesAreLazy;
+        }
+
+        let pathValues: PathValue[] = [];
+        for (let i = 0; i < settings.valueCount; i++) {
+            let partialValue = partialValues[i];
+            pathValues.push({
+                path: partialValue.path,
+                time: partialValue.time,
+
+                locks: readLocks[i],
+                value: values[i],
+                isValueLazy: valuesAreLazy[i] || false,
+
+                canGCValue: partialValue.canGCValue,
+                event: partialValue.event,
+                isTransparent: partialValue.isTransparent,
+                valid: partialValue.valid,
+                source: partialValue.source,
+                lockCount: partialValue.lockCount,
+                updateCount: partialValue.updateCount,
+                transactionPaths: transactionPaths[i],
+            });
+        }
+
+        return pathValues;
+    }
+
     lazyValues = new WeakMap<{}, Buffer>();
     addLazyValue(buf: Buffer) {
         // NOTE: Using an object makes garbage collection MUCH easier, basically eliminating leaks (and...

package/src/0-path-value-core/pathValueArchives.ts

@@ -128,7 +128,13 @@ export class PathValueArchives {
             values = await this.filterForDiskStorage(values);
         }
 
-        let buffers = await pathValueSerializer.serialize(values, { noLocks: true, compress: getCompressNetwork(), singleBuffer: true });
+        let buffers = await pathValueSerializer.serialize(values, {
+            noLocks: true,
+            compress: getCompressNetwork(),
+            singleBuffer: true,
+            // NOTE: The system natively handles the concept of knowing not to do a commit if it doesn't have any value. The only reason transactions are required is we might have a value, but it might not be the latest value. However, if we're writing to the disk, we're not going to have an even older value. Anything we write to the disk is going to be the oldest value that we'll receive.
+            noTransactionPaths: true
+        });
         let data = buffers[0];
 
         let minTime = values[0].time.time;

package/test.ts

@@ -7,7 +7,7 @@ import { getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
 import { green, yellow } from "socket-function/src/formatting/logColors";
 import { Querysub, t } from "./src/4-querysub/Querysub";
-import { pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
+import { archives, pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
 import { getAllAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 import { deploySchema } from "./src/4-deploy/deploySchema";
 import { getDomain } from "./src/config";
@@ -17,6 +17,8 @@ import { RemoteWatcher } from "./src/1-path-client/RemoteWatcher";
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { shutdown } from "./src/diagnostics/periodic";
 import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
+import { PathValue, epochTime } from "./src/0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
 
 let tempTestSchema = Querysub.createSchema({
     value: t.number,
@@ -28,13 +30,14 @@ let tempTestSchema = Querysub.createSchema({
 });
 
 async function main() {
-
-    let prefixes = await getShardPrefixes();
-    for (let prefix of prefixes) {
-        console.log(prefix);
-    }
     // await Querysub.hostService("test");
 
+    // let testValues: PathValue[] = [];
+    // let buffers = await pathValueSerializer.serialize(testValues);
+    // let values = await pathValueSerializer.deserialize(buffers);
+
+    let values = await pathValueArchives.loadValues(await getAllAuthoritySpec());
+
     // let path = getProxyPath(() => tempTestSchema.data().value);
     // console.log({ path });
     // let authorities = PathRouter.getAllAuthorities(path);