querysub 0.2.0

package/src/0-path-value-core/pathValueArchives.ts

@@ -0,0 +1,483 @@
+import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_FLUSH_LIMIT_START, compareTime, debugPathValuePath, getCompressNetwork, isCoreQuiet, PathValue, PathValueSnapshot, registerGetCompressNetwork, Time } from "./pathValueCore";
+import { getArchives, nestArchives } from "../-a-archives/archives";
+import { cache, cacheLimited, lazy } from "socket-function/src/caching";
+import { getOwnNodeId, getOwnNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import debugbreak from "debugbreak";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { list } from "socket-function/src/misc";
+import { AuthorityPath, pathValueAuthority2 } from "./NodePathAuthorities";
+import { NodeIdParts, decodeNodeId, decodeNodeIdAssert, encodeNodeId } from "../-a-auth/certs";
+import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
+import { devDebugbreak, isNoNetwork } from "../config";
+import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
+
+const archives = lazy(() => wrapArchivesWithCache(getArchives("path-values/")));
+const archivesLocks = lazy(() => getArchives("path-values-locks/"));
+const archivesRecycleBin = lazy(() => getArchives("path-values-recycle-bin/"));
+
+export type RawValuePaths = {
+    dataPath: string;
+    values: PathValue[];
+    error?: string;
+};
+export type DecodedValuePath = NodeIdParts & {
+    // Path is the original path we decoded this from.
+    path: string;
+
+    time: number;
+    seqNum: number;
+    valueCount: number;
+    byteCount: number;
+    minTime: number | undefined;
+    maxTime: number | undefined;
+
+    sourceType: "genesis" | "merge" | "gc" | string;
+    // misc can be anything that doesn't include "_"
+    //  - AND, either was need lastMoved, OR, it needs to not decode with decodeNodeId
+    misc?: string;
+};
+export type DecodedValuePathInput = Omit<DecodedValuePath, "path">;
+function defaultDecodedValuePath(path: string): DecodedValuePath {
+    return {
+        path,
+        time: 0, seqNum: 0, valueCount: 0, byteCount: 0, threadId: "", domain: "", machineId: "", port: -1,
+        maxTime: undefined, minTime: undefined,
+        sourceType: "genesis",
+    };
+}
+
+
+
+export class PathValueArchives {
+    nextWriteSeqNum = 1;
+
+    // NOTE: We only archive the latest value for each path, and only read the latest. This means that all
+    //  archives values MUST be golden (valid and as old as the max change state), otherwise the archives might
+    //  turn invalid values into valid values.
+
+    @measureFnc
+    public decodeDataPathBatch(dataPath: string[]) {
+        return dataPath.map(x => this.decodeDataPath(x));
+    }
+    public decodeDataPath(path: string): DecodedValuePath {
+        let originalPath = path;
+        if (path.endsWith(".locked")) {
+            path = path.slice(0, -".locked".length);
+        }
+        if (!path.endsWith(".data")) return defaultDecodedValuePath(path);
+        let dataPath = path.slice(0, -".data".length);
+        let fileName = dataPath.replaceAll("\\", "/").split("/").at(-1)!;
+        let parts = fileName.split("_");
+        if (parts[0] !== "data") return defaultDecodedValuePath(dataPath);
+        let nodeId = parts[1];
+        let decodedNodeId = decodeNodeId(nodeId);
+        if (!decodedNodeId) return defaultDecodedValuePath(dataPath);
+        let time = +parts[2] || 0;
+        let seqNum = +parts[3] || 0;
+        let valueCount = +parts[4] || 0;
+        let byteCount = +parts[5] || 0;
+        let minTime = +parts[6] || 0;
+        let maxTime = +parts[7] || 0;
+        let sourceType = parts[8] as DecodedValuePath["sourceType"] || "genesis";
+        let misc = parts[9];
+        return {
+            path: originalPath,
+            ...decodedNodeId,
+            minTime, maxTime,
+            time, seqNum, valueCount, byteCount,
+            sourceType,
+            misc,
+        };
+    }
+    public encodeDataPath(decoded: DecodedValuePathInput) {
+        return `data_${encodeNodeId(decoded)}_${decoded.time}_${decoded.seqNum}_${decoded.valueCount}_${decoded.byteCount}_${decoded.minTime}_${decoded.maxTime}_${decoded.sourceType}_${decoded.misc || ""}.data`;
+    }
+
+    public getDefaultValuePathInput(): DecodedValuePathInput {
+        return {
+            ...decodeNodeIdAssert(getOwnNodeIdAssert()),
+            time: Date.now(),
+            seqNum: this.nextWriteSeqNum++,
+            valueCount: 0,
+            byteCount: 0,
+            minTime: Date.now(), maxTime: Date.now(),
+            sourceType: "genesis",
+        };
+    }
+
+    public async encodeValuePaths(
+        values: PathValue[],
+        config?: {
+            pathOverrides?: Partial<DecodedValuePathInput>,
+            // Don't dedupe values
+            noFilter?: boolean;
+        }
+    ) {
+        let time = Date.now();
+
+        if (!isCoreQuiet) {
+            console.log(blue(`Serializing ${values.length} ValuePaths`));
+        }
+        if (values.length === 0) return undefined;
+
+        if (!config?.noFilter) {
+            values = await this.filterForDiskStorage(values);
+        }
+
+        let buffers = await pathValueSerializer.serialize(values, { noLocks: true, compress: getCompressNetwork(), singleBuffer: true });
+        let data = buffers[0];
+
+        let minTime = values[0].time.time;
+        let maxTime = values[0].time.time;
+        for (let value of values) {
+            let time = value.time.time;
+            if (time < minTime) minTime = time;
+            if (time > maxTime) maxTime = time;
+        }
+
+        // NOTE: Due to merging storing firstTime/lastTime won't be overly useful here, as we should shard enough
+        //      that nodes only have a few data files (and so the time extent would be quickly become the
+        //      entire available time range).
+        let file = this.encodeDataPath({
+            ...this.getDefaultValuePathInput(),
+            valueCount: values.length,
+            byteCount: data.byteLength,
+            minTime, maxTime,
+            sourceType: "genesis",
+            ...config?.pathOverrides,
+        });
+
+        if (!isCoreQuiet) {
+            time = Date.now() - time;
+            console.log(blue(`Finished serializing ${values.length} in ${formatTime(time)} for ${file}`));
+        }
+
+        return { file, data };
+    }
+
+
+    @measureFnc
+    public async archiveValues(pathAuthority: AuthorityPath, values: PathValue[]) {
+        let authorityPath = pathValueAuthority2.getArchiveDirectory(pathAuthority);
+
+        let encodedObj = await this.encodeValuePaths(values);
+        if (!encodedObj) return;
+        let { file, data } = encodedObj;
+
+        let time = Date.now();
+
+        let maxAttemptArchiveTime = Date.now() - ARCHIVE_FLUSH_LIMIT;
+
+        let oldestTime = Number.MAX_SAFE_INTEGER;
+        for (let value of values) {
+            let time = value.time.time;
+            // We should be ignoring values WAY before this on receive. We can't archive it now,
+            //  because even though not archiving it will lose data, if we do archive it we will
+            //  break existing servers (and it could cause security issues, etc).
+            if (time < maxAttemptArchiveTime) {
+                devDebugbreak();
+                console.error(`Tried to archive a value which is too far into the past, ${time} < ${maxAttemptArchiveTime}. To commit this to disk would break locks, and could leave the database in an invalid state. Killing server so clients will resync with a server that is in a better state.`);
+                process.exit();
+            }
+            if (time < oldestTime) oldestTime = time;
+        }
+        let slowestFileWriteTime = oldestTime + ARCHIVE_FLUSH_LIMIT;
+        await archives().set(authorityPath + file, data);
+
+        let fileInfo = await archives().getInfo(authorityPath + file);
+        // NOTE: If no fileInfo... then our file was merged? Which... is BAD, as it means we took
+        //      too long to read it, so we probably took too long to write it too!
+        if (!fileInfo || fileInfo.writeTime > slowestFileWriteTime) {
+            console.error(red(`File ${authorityPath + file} was written too slowly, ${fileInfo?.writeTime || "undefined"} < ${slowestFileWriteTime}. This means some values will be rejected by reads. Killing server, our state is irrecoverable. Our watches have invalid data, and we have to stop before we create more invalid dependencies.`));
+            process.exit();
+        }
+
+
+        if (!isCoreQuiet) {
+            time = Date.now() - time;
+            console.log(blue(`Finished archiving ${values.length} to ${authorityPath} in ${formatTime(time)}`));
+        }
+    }
+
+    @measureFnc
+    public async loadValues(authority: AuthorityPath): Promise<PathValueSnapshot> {
+
+
+        let dirName = pathValueAuthority2.getArchiveDirectory(authority);
+        console.log(blue(`Loading data ${dirName} at ${new Date().toISOString()}`));
+
+        // let testFileName = getOwnNodeId() + "_" + Date.now() + "_" + 1_000_000_000;
+        // await archives().assertPathValid(dirName + "data_" + testFileName + ".data");
+
+        let snapshot: PathValueSnapshot = {
+            values: {},
+        };
+        let totalCount = 0;
+        let uniqueCount = 0;
+        let fileCount = 0;
+        let totalSize = 0;
+
+        let dataPaths: string[] = [];
+        let readCache = new Map<string, Buffer>();
+
+        let downloadStartTime = Date.now();
+        while (true) {
+            // NOTE: Getting all data paths FIRST is important. Otherwise files moves (even ones which take
+            //  into account readers by copying, waiting a minute, and then deleting) can cause us to miss data
+            //  (we read the target dest, find no file, read a bunch of other files over a few minutes, get to
+            //  the source file, but by now it has moved, so we don't see either file, even though at all times
+            //  both files existed).
+            dataPaths = await this.getValuePaths(authority);
+            console.log(blue(`    ${dataPaths.length} data paths`));
+            // NOTE: If the notMatched count is high enough, it is possible NodePathAuthorities.ts:authoritiesMightOverlap is
+            //  too loose, and should be removing more cases.
+
+            let pendingDataPaths = dataPaths.slice();
+            // We CAN'T just read everything at once, as this will cause us to
+            //      run out of file handles, and error out.
+            async function runReadThread() {
+                while (pendingDataPaths.length > 0) {
+                    let dataPath = pendingDataPaths.pop()!;
+                    if (readCache.has(dataPath)) continue;
+                    let data = await archives().get(dataPath);
+                    if (!data) continue;
+                    readCache.set(dataPath, data);
+                }
+            }
+            await Promise.all(list(32).map(runReadThread));
+
+            let files = dataPaths.map(x => readCache.get(x));
+            if (files.some(x => !x)) {
+                let missingFiles = dataPaths.filter(x => !readCache.has(x));
+                // This likely means a merge happened while we were reading the files. If we just ignore the
+                //      missing files, we will almost certainly miss the merge output files, which will cause us
+                //      to miss a lot of data, and be in an entirely broken state.
+                console.log(red("Some files are missing, rereading directory and trying again"));
+                for (let missingFile of missingFiles) {
+                    console.log(red(`    ${missingFile}`));
+                }
+                continue;
+            }
+            break;
+        }
+        console.log(green(`Downloaded/read ${dataPaths.length} data paths in ${formatTime(Date.now() - downloadStartTime)}`));
+
+        let parseTime = Date.now();
+        let notMatched = 0;
+        let tooOldValues = 0;
+
+        for (let dataFile of dataPaths) {
+            let data = readCache.get(dataFile);
+            if (!data) throw new Error(`Data file ${dataFile} not in lookup which we just checked?`);
+            totalSize += data.byteLength;
+            fileCount++;
+            let loadObj = await PathValueArchives.loadValuesFromBuffer({ path: dataFile, data });
+            let dataValues = loadObj.values;
+            tooOldValues += loadObj.tooOldValues;
+            totalCount += loadObj.rawCount;
+
+            for (let value of dataValues) {
+                // NOTE: We can't return extra values, as they might be outdated values which were
+                //  set to undefined, and then deleted, so returning them will create spurious values.
+                if (!pathValueAuthority2.isInAuthority(authority, value.path)) {
+                    notMatched++;
+                    continue;
+                }
+                let path = value.path;
+                if (!snapshot.values[path]) {
+                    uniqueCount++;
+                    snapshot.values[path] = [value];
+                } else {
+                    if (compareTime(value.time, snapshot.values[path][0].time) >= 0) {
+                        snapshot.values[path][0] = value;
+                    }
+                }
+            }
+        }
+
+        let relevantCount = totalCount - notMatched;
+        parseTime = Date.now() - parseTime;
+        console.log((`${green("Finished parsing")} ${formatNumber(totalCount)} in ${formatNumber(totalSize)}B (${formatNumber(uniqueCount / relevantCount * 100)}% unique paths = ${formatNumber(uniqueCount)}), in ${green(formatTime(parseTime))}, ${fileCount} files, authority ${magenta(dirName)}, (${formatNumber(notMatched / totalCount * 100)}% ignored, due to being for another authority) at ${new Date().toISOString()}. Startup time ${green(formatTime(process.uptime() * 1000))}`));
+
+        if (tooOldValues) {
+            console.warn(yellow(`Discarded ${formatNumber(tooOldValues)} values discarded for being part of hanging server writes`));
+        }
+
+        return {
+            values: snapshot.values,
+        };
+    }
+
+    public static async loadValuesFromBuffer(config: {
+        path: string;
+        data: Buffer;
+        skipStrings?: boolean;
+        skipValues?: boolean;
+    }): Promise<{
+        values: PathValue[];
+        rawCount: number;
+        tooOldValues: number;
+    }> {
+        const { path, data } = config;
+        let tooOldValues = 0;
+
+        let dataValues: PathValue[] = [];
+
+        try {
+            dataValues = await pathValueSerializer.deserialize([data], {
+                singleBuffer: true,
+                skipStrings: config.skipStrings,
+                skipValues: config.skipValues,
+            });
+        } catch (e: any) {
+            console.log(red(`Bad archive data file at ${config.path}, error: ${e.stack}`));
+        }
+        let rawCount = dataValues.length;
+
+        let fileObj = pathValueArchives.decodeDataPath(path);
+        if (
+            fileObj.sourceType === "genesis"
+            && fileObj.time > ARCHIVE_FLUSH_LIMIT_START
+        ) {
+            let oldestAllowedTime = fileObj.time - ARCHIVE_FLUSH_LIMIT_START;
+            dataValues = dataValues.filter(x => {
+                if (x.time.time < oldestAllowedTime) {
+                    tooOldValues++;
+                    return false;
+                }
+                return true;
+            });
+        }
+
+        return {
+            values: dataValues,
+            rawCount,
+            tooOldValues,
+        };
+    }
+
+    public async getRawValueFiles(dataPath: string[], config?: {
+        includeRecycleBin?: boolean;
+    }): Promise<(Buffer | undefined)[]> {
+        // let results: (Buffer | undefined)[] = [];
+        // let sum = 0;
+        // for (let path of dataPath) {
+        //     console.log(blue(`Getting ${path}`));
+        //     let result = await archives().get(path);
+        //     results.push(result);
+        //     console.log(blue(`Finished getting ${path}, size ${formatNumber(result?.byteLength || 0)}`));
+        //     sum += result?.byteLength || 0;
+        // }
+        // console.log(blue(`Total size ${formatNumber(sum)}`));
+        // return results;
+        let results: (Buffer | undefined)[] = [];
+        const BATCH_SIZE = 64;
+        for (let i = 0; i < dataPath.length; i += BATCH_SIZE) {
+            let cur = dataPath.slice(i, i + BATCH_SIZE);
+            let curResults = await Promise.all(cur.map(x => archives().get(x)));
+            results.push(...curResults);
+        }
+        if (config?.includeRecycleBin) {
+            let notFoundIndexes = new Map<number, string>();
+            for (let i = 0; i < dataPath.length; i++) {
+                if (!results[i]) {
+                    notFoundIndexes.set(i, dataPath[i]);
+                }
+            }
+            let notFoundList = Array.from(notFoundIndexes.keys());
+            for (let i = 0; i < notFoundList.length; i += BATCH_SIZE) {
+                let curIndexes = notFoundList.slice(i, i + BATCH_SIZE);
+                let cur = curIndexes.map(x => dataPath[x]);
+                let curResults = await Promise.all(cur.map(x => archivesRecycleBin().get(x)));
+                for (let j = 0; j < curResults.length; j++) {
+                    results[curIndexes[j]] = curResults[j];
+                }
+            }
+        }
+        return results;
+    }
+
+    public async getArchiveLocker() {
+        // return createArchiveLocker({
+        //     archiveValues: archives(),
+        //     archiveRecycleBin: archivesRecycleBin(),
+        //     archiveLocks: archivesLocks(),
+        // });
+        return createArchiveLocker2({
+            archiveValues: archives(),
+            archiveRecycleBin: archivesRecycleBin(),
+            archiveLocks: archivesLocks(),
+        });
+    }
+    @measureFnc
+    public async getAuthorityDirs(authority: AuthorityPath): Promise<string[]> {
+        let allAuthorityDirectories = await archives().find("", { shallow: true, type: "folders" });
+
+        let pickedDirectories = new Set<string>();
+        const authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
+        pickedDirectories.add(authorityDir);
+        for (let filePath of allAuthorityDirectories) {
+            if (pathValueAuthority2.mightMatchArchiveDirectory(authority, filePath)) {
+                pickedDirectories.add(filePath);
+            }
+        }
+
+        return Array.from(pickedDirectories);
+    }
+
+    @measureFnc
+    public async getValuePaths(authority: AuthorityPath): Promise<string[]> {
+        // let paths: string[] = [];
+        // let pickedDirectories = await this.getAuthorityDirs(authority);
+
+        // for (let authorityFilePath of pickedDirectories) {
+        //     let dataFiles = await archives().find(authorityFilePath + "/data_", { shallow: true, type: "files" });
+        //     for (let dataFile of dataFiles) {
+        //         paths.push(dataFile);
+        //     }
+        // }
+
+        let locker = await this.getArchiveLocker();
+        let allFiles = (await locker.getAllValidFiles()).map(x => x.file);
+        return allFiles.filter(x => pathValueAuthority2.mightMatchArchiveDirectory(authority, x));
+    }
+    @measureFnc
+    public async getValuePathSizes(paths: string[]): Promise<number[]> {
+        return await Promise.all(paths.map(async path => {
+            let info = await archives().getInfo(path);
+            // if (!info) {
+            //     let value = await archives().get(path);
+            //     if (value) {
+            //         debugbreak(2);
+            //         debugger;
+            //         await archives().getInfo(path);
+            //         console.log(info, value);
+            //     }
+            // }
+            return info?.size || 0;
+        }));
+    }
+
+
+    @measureFnc
+    private filterForDiskStorage(values: PathValue[]): PathValue[] {
+        let latestValues = new Map<string, PathValue>();
+        for (let value of values) {
+            if (!value.valid) {
+                console.error(`Only valid values should be archived. Tried to archive invalid value ${debugPathValuePath(value)}`);
+                continue;
+            }
+            let prev = latestValues.get(value.path);
+            if (prev && compareTime(value.time, prev.time) < 0) continue;
+            latestValues.set(value.path, value);
+        }
+
+        values = Array.from(latestValues.values());
+
+        return values;
+    }
+}
+export const pathValueArchives = new PathValueArchives();