querysub 0.2.0

package/src/2-proxy/archiveMoveHarness.ts

@@ -0,0 +1,376 @@
+
+
+// yarn typenode src\framework-beta\archiveMoveBase\archiveMoveHarness.ts --nonetwork
+
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { blue, green, red } from "socket-function/src/formatting/logColors";
+import { measureBlock, measureWrap } from "socket-function/src/profiling/measure";
+import { AuthorityPath, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { FileInfo, ArchiveTransaction } from "../0-path-value-core/archiveLocks/ArchiveLocks";
+import { DecodedValuePath, pathValueArchives, PathValueArchives } from "../0-path-value-core/pathValueArchives";
+import { PathValue, VALUE_GC_THRESHOLD, FILE_VALUE_COUNT_LIMIT, FILE_SIZE_LIMIT, compareTime } from "../0-path-value-core/pathValueCore";
+import { logNodeStats } from "../5-diagnostics/nodeMetadata";
+import { getSingleSizeEstimate } from "../5-diagnostics/shared";
+import { getOurAuthorities } from "../config2";
+import debugbreak from "debugbreak";
+
+// NOTE: We probably COULD load values with skipStrings+skipValues, for some GCers, as some
+//      GCing is as simple as, "if this value is undefined, delete it". But... we GC
+//      so seldomly, that it wouldn't make a difference (going from 1 minute of GC time
+//      per day to 20 seconds doesn't really matter).
+
+let lastFileWrites = new Map<string, string[]>();
+
+export async function runArchiveMover(config: {
+    // Defaults to getOurAuthorities
+    //  NOTE: Runs once per authority, as we want to keep data as sharded as possible
+    authorities?: AuthorityPath[];
+
+    // If set we will provide all values. This means this is a high chance out of order values will
+    //      be provided. Otherwise a consistent view of data at Date.now() - ARCHIVE_FLUSH_LIMIT
+    //      (but is quite old, so data won't appear for around an hour).
+    readLiveData?: boolean;
+
+    // Runs without locking the existence of files, so duplicates can appear. Results in less rejections,
+    //  but... then you might also have duplicate (or even redundant but slightly different) files.
+    allowDuplicates?: boolean;
+
+    maxFileValueCount?: number;
+    maxFileByteCount?: number;
+
+    /** We abort the move if the new count > inputFile * newFileCountThreshold.
+     *      - runArchiveMover CANNOT add new data, and is only designed to remove unneeded data.
+     *          Due to how caching works, if we aren't removing enough files, or enough bytes,
+     *          (for example, if we reduce the size by 10%), then the move will likely result in
+     *          MORE data being read, and so isn't worth it.
+     *  (LOWER means we merge less often)
+     */
+    newFileCountThreshold?: number;
+    /** We abort the move if the new bytes > inputFile * newFileSizeThreshold
+     *  (LOWER means we merge less often)
+    */
+    newFileSizeThreshold?: number;
+
+    /** Always commit changes, even if the thresholds are not met. */
+    force?: boolean;
+
+    outputType: DecodedValuePath["sourceType"];
+    runMover: (
+        config: {
+            values: PathValue[];
+            /** We provide all values on authority, that are <= snapshotTime, that will ever exist
+             *      (it's an atomic snapshot).
+             *      - Of course, if readLiveData is set this is Number.POSITIVE_INFINITY, as we
+             *          try to provide the latest data... making this no longer an atomic snapshot.
+             */
+            authority: AuthorityPath;
+            snapshotTime: number;
+            logKey: string;
+        }
+    ) => Promise<{
+        newValues: "abort" | {
+            // A file is created per key (with the key somewhere in the file name, although perhaps escaped or hashed).
+            //  We might split one file further, if the size exceeds certain limits. These limits are mostly
+            //      for diagnostics, to allow us to view snapshots by time.
+            [key: string]: PathValue[];
+        };
+    }>;
+}) {
+    let authorities = config.authorities ?? getOurAuthorities();
+    let minAge = config.readLiveData ? Number.NEGATIVE_INFINITY : VALUE_GC_THRESHOLD;
+    let minThreshold = Date.now() - minAge;
+    // NOTE: If the value count is too high, parsing in the ArchiveViewer might crash the browser.
+    const maxFileValueCount = config.maxFileValueCount ?? FILE_VALUE_COUNT_LIMIT;
+
+    const maxFileByteCount = config.maxFileByteCount ?? FILE_SIZE_LIMIT;
+    // NOTE: The file count threshold is quite high, to prevent the genesis writes (even though they
+    //      are merged already) from causing us to re-merge all the files unnecessarily.
+    const newFileCountThreshold = config.newFileCountThreshold ?? (1 / 10);
+    const newFileSizeThreshold = config.newFileSizeThreshold ?? 0.6;
+    function meetsThreshold(config: {
+        inputFiles: number;
+        inputBytes: number;
+        outputFiles: number;
+        outputBytes: number;
+    }) {
+        let { inputFiles, inputBytes, outputFiles, outputBytes } = config;
+        let outputThreshold = Math.ceil(inputFiles * newFileCountThreshold);
+        let outputSizeThreshold = Math.ceil(inputBytes * newFileSizeThreshold);
+        let meets = (outputFiles < outputThreshold || outputBytes < outputSizeThreshold);
+        if (!meets) {
+            console.log(`Skipping move due to insufficient reduction. Thresholds not met. Either files ${outputFiles} < ${outputThreshold} OR ${formatNumber(outputBytes)}B < ${formatNumber(outputSizeThreshold)}B, and neither are.`);
+        }
+        return meets;
+    }
+
+    // NOTE: We COULD read from all authorities, handling them all at once. This MIGHT making
+    //      gcing more efficient. But... it also might not. We'll see.
+    //      - And, sharding shouldn't be random, it should be path based, just like GCing. Although,
+    //          if we have references, the best we can do is add parallel authorities to nodes,
+    //          so... we might have to handle all authorities at once, instead of looping like this...
+    for (let authority of authorities) {
+        let locker = await pathValueArchives.getArchiveLocker();
+        let matchedDirs = await pathValueArchives.getAuthorityDirs(authority);
+        let authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
+
+        let startTime = Date.now();
+        let readCache = new Map<string, Buffer>();
+        let lastAborted = false;
+        let lastResult: "accepted" | "rejected" | "aborted" = "accepted";
+
+        let inputBytes = 0;
+        let outputBytes = 0;
+        let inputValueCount = 0;
+        let outputValueCount = 0;
+        let inputFiles = 0;
+        let outputFiles = 0;
+        let abortedDueToInsufficientReduction = false;
+
+        let reread = true;
+        while (reread) {
+            reread = false;
+            lastAborted = false;
+            lastResult = await locker.atomicSwapFiles({
+                allowDuplicates: config.allowDuplicates,
+            }, async (valueFiles, readFiles) => {
+                valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
+
+                valueFiles = valueFiles.filter(x => {
+                    let obj = pathValueArchives.decodeDataPath(x.file);
+                    if (!obj.minTime) return true;
+                    return !(obj.minTime > minThreshold);
+                });
+
+                let lastWriteHash = JSON.stringify(matchedDirs);
+                // NOTE: ACTUALLY, this doesn't work, as the oldFiles might values which have GC timers,
+                //      or which... are large, and just became deleted.
+                // let lastFiles = new Set(lastFileWrites.get(lastWriteHash) || []);
+                // let newFiles = valueFiles.filter(x => !lastFiles.has(x.file));
+                // // BEST case, ALL oldFiles are collapsed. And if that doesn't even cause a move,
+                // //  then just skip the check now
+                // if (!config.force && meetsThreshold({
+                //     inputFiles: valueFiles.length,
+                //     inputBytes: valueFiles.map(x => x.size).reduce((a, b) => a + b, 0),
+                //     outputFiles: newFiles.length,
+                //     outputBytes: newFiles.map(x => x.size).reduce((a, b) => a + b, 0),
+                // })) {
+                //     return [];
+                // }
+
+                // Read files, and if any are missing, rerun until we can read all files
+                await measureBlock(async function locker_readFiles() {
+                    let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
+                    let buffers = await readFiles(remainingFiles);
+                    for (let i = 0; i < remainingFiles.length; i++) {
+                        let buffer = buffers[i];
+                        if (!buffer) {
+                            console.log(`File missing, re-reading`, remainingFiles[i].file);
+                            reread = true;
+                            break;
+                        }
+                        readCache.set(remainingFiles[i].file, buffer);
+                    }
+                });
+                if (reread) {
+                    console.log(`Files changed, re-reading`);
+                    return [];
+                }
+                function isValueIncluded(value: PathValue) {
+                    if (value.time.time > minThreshold) return false;
+                    if (!pathValueAuthority2.isInAuthority(authority, value.path)) return false;
+                    return true;
+                }
+
+                let mutatedFiles: FileInfo[] = [];
+                // authorityFolder => values
+                let floatingValues = new Map<string, PathValue[][]>();
+                let inputValues: PathValue[][] = [];
+                inputBytes = 0;
+                inputFiles = 0;
+                inputValueCount = 0;
+
+                await measureBlock(async function locker_parseFiles() {
+                    for (let file of valueFiles) {
+                        let buffer = readCache.get(file.file);
+                        if (!buffer) {
+                            throw new Error(`Buffer missing for ${file.file}`);
+                        }
+                        inputBytes += buffer.byteLength;
+                        inputFiles++;
+                        let valuesObj = await PathValueArchives.loadValuesFromBuffer({
+                            path: file.file,
+                            data: buffer,
+                        });
+                        let values = valuesObj.values;
+                        inputValueCount += values.length;
+                        let includedValues: PathValue[] = [];
+                        let excludedValues: PathValue[] = [];
+                        for (let value of values) {
+                            if (isValueIncluded(value)) {
+                                includedValues.push(value);
+                            } else {
+                                excludedValues.push(value);
+                            }
+                        }
+                        if (includedValues.length === 0) {
+                            continue;
+                        }
+                        if (excludedValues.length > 0) {
+                            let fileDir = file.file.split("/").slice(0, -1).join("/") + "/";
+                            let list = floatingValues.get(fileDir);
+                            if (!list) {
+                                floatingValues.set(fileDir, list = []);
+                            }
+                            list.push(excludedValues);
+                        }
+                        inputValues.push(includedValues);
+                        mutatedFiles.push(file);
+                    }
+                });
+
+                let inputValuesFlat = inputValues.flat();
+                let result = await measureBlock(async function locker_runApp() {
+                    return config.runMover({
+                        values: inputValuesFlat,
+                        authority,
+                        snapshotTime: minThreshold,
+                        logKey: authorityDir,
+                    });
+                }, `locker_app_${config.runMover.name}`);
+                if (result.newValues === "abort") {
+                    lastAborted = true;
+                    return [];
+                }
+
+                let transaction: ArchiveTransaction = {
+                    createFiles: [],
+                    deleteFiles: mutatedFiles,
+                };
+                const addValues = measureWrap(async function locker_addValues(dir: string, values: PathValue[], key: string) {
+                    let split = splitValuesBelowLimit({
+                        values,
+                        maxFileValueCount,
+                        maxFileByteCount,
+                    });
+                    outputFiles += split.length;
+                    outputValueCount += values.length;
+                    for (let splitValue of split) {
+                        let dataObj = await pathValueArchives.encodeValuePaths(splitValue.values, {
+                            pathOverrides: {
+                                sourceType: config.outputType,
+                                misc: key,
+                            },
+                        });
+                        if (!dataObj) continue;
+                        outputBytes += dataObj.data.byteLength;
+                        let fileName = pathValueArchives.encodeDataPath({
+                            ...pathValueArchives.getDefaultValuePathInput(),
+                            valueCount: splitValue.values.length,
+                            byteCount: dataObj.data.byteLength,
+                            minTime: splitValue.minTime,
+                            maxTime: splitValue.maxTime,
+                            sourceType: config.outputType,
+                            misc: key,
+                        });
+                        transaction.createFiles.push({
+                            data: dataObj.data,
+                            file: dir + fileName,
+                        });
+                    }
+                });
+
+                for (let [authorityDir, values] of floatingValues) {
+                    await addValues(authorityDir, values.flat(), "floating");
+                }
+
+                for (let key in result.newValues) {
+                    let values = result.newValues[key];
+                    await addValues(authorityDir, values, key);
+                }
+
+                if (!config.force && !meetsThreshold({ inputFiles, inputBytes, outputFiles, outputBytes })) {
+                    abortedDueToInsufficientReduction = true;
+                    return [];
+                }
+
+                lastFileWrites.set(lastWriteHash, transaction.createFiles.map(x => x.file));
+
+                return [transaction];
+            });
+            if (lastAborted) {
+                lastResult = "aborted";
+            }
+        }
+        let time = Date.now() - startTime;
+        let resultFormat = (
+            lastResult === "accepted" && green("accepted")
+            || lastResult === "aborted" && green("noop")
+            || red("rejected")
+        );
+        console.log(`ArchiveMover ${config.runMover.name} on ${authorityDir} in ${formatTime(time)}. Result = ${resultFormat}.`);
+        if (lastResult === "accepted" && !abortedDueToInsufficientReduction) {
+            console.log(
+                `    { files: ${formatNumber(inputFiles)}, values: ${formatNumber(inputValueCount)}, bytes: ${formatNumber(inputBytes)} }`
+                + "\n=>\n"
+                + `    { files: ${formatNumber(outputFiles)}, values: ${formatNumber(outputValueCount)}, bytes: ${formatNumber(outputBytes)} }`
+            );
+            logNodeStats("archives|Input Files", formatNumber)(time);
+            logNodeStats("archives|Input Values", formatNumber)(time);
+            logNodeStats("archives|Input Bytes", formatNumber)(time);
+            logNodeStats("archives|Output Files", formatNumber)(time);
+            logNodeStats("archives|Output Values", formatNumber)(time);
+            logNodeStats("archives|Output Bytes", formatNumber)(time);
+        }
+        console.log(" ");
+    }
+}
+
+const splitValuesBelowLimit = measureWrap(function splitValuesBelowLimit(config: {
+    values: PathValue[];
+    maxFileValueCount: number;
+    maxFileByteCount: number;
+}): {
+    values: PathValue[];
+    minTime: number;
+    maxTime: number;
+}[] {
+    let { values, maxFileValueCount, maxFileByteCount } = config;
+    values = values.slice();
+    values.sort((a, b) => compareTime(a.time, b.time));
+
+    let output: {
+        values: PathValue[];
+        minTime: number;
+        maxTime: number;
+    }[] = [];
+
+    let pending: {
+        values: PathValue[];
+        byteCount: number;
+    } = {
+        values: [],
+        byteCount: 0,
+    };
+    function addPending() {
+        if (pending.values.length === 0) return;
+        let vals = pending.values;
+        output.push({
+            values: vals,
+            minTime: vals[0].time.time,
+            maxTime: vals[vals.length - 1].time.time,
+        });
+        pending.values = [];
+        pending.byteCount = 0;
+    }
+    for (let value of values) {
+        let byteCount = getSingleSizeEstimate(value);
+        if (pending.values.length >= maxFileValueCount || pending.byteCount + byteCount >= maxFileByteCount) {
+            addPending();
+        }
+        pending.values.push(value);
+        pending.byteCount += byteCount;
+    }
+    addPending();
+
+    return output;
+});