querysub 0.403.0 → 0.405.0

package/src/0-path-value-core/ValidStateComputer.ts

@@ -0,0 +1,303 @@
+import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { isNode } from "typesafecss";
+import { isDiskAudit } from "../config";
+import { getPathFromStr } from "../path";
+import { auditLog } from "./auditLogs";
+import { PathRouter } from "./PathRouter";
+import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue } from "./pathValueCore";
+import { pathWatcher } from "./PathWatcher";
+import { lockWatcher2 } from "./LockWatcher2";
+import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
+
+class ValidStateComputer {
+
+    private capturePrevValidStates(valuePaths: PathValue[]): Map<PathValue, boolean | undefined> {
+        let prevValidStates = new Map<PathValue, boolean | undefined>();
+        for (let value of valuePaths) {
+            let prevValidState = authorityStorage.getValueAtTime(value.path, value.time)?.valid;
+            prevValidStates.set(value, prevValidState);
+        }
+        return prevValidStates;
+    }
+
+    @measureFnc
+    public ingestValuesAndValidStates(config: {
+        pathValues: PathValue[];
+        parentSyncs: { parentPath: string; sourceNodeId: string }[];
+        initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+        doNotArchive?: boolean;
+    }) {
+        let { pathValues, parentSyncs, initialTriggers, doNotArchive } = config;
+
+        // TODO: We might want to add back optimizations for "no watches and no locks"?
+        // TODO: If we find it is a serious performance problem, we could just send the valid state instead of sending the full path value and values are rejected (this is what we used to do).
+
+        let now = Date.now();
+
+        authorityStorage.addParentSyncs(parentSyncs);
+
+        let ourValues: PathValue[] = [];
+        let notOurValues: PathValue[] = [];
+        for (let value of pathValues) {
+            if (!PathRouter.isSelfAuthority(value.path)) {
+                notOurValues.push(value);
+            } else {
+                ourValues.push(value);
+            }
+        }
+        authorityStorage.ingestValues(notOurValues, { doNotArchive });
+
+
+        // Use initial triggers, at least for all the values that we aren't the authority of, to clobber the old values, creating new shallow copies of the old values with a valid state false if we don't have the corresponding value in our new values, including for all parent paths. 
+        {
+            // TODO: This code is similar to the code in trigger values change, but it's also dealing with watchers and other stuff, so I think it makes sense to have it also here.
+            let fullInitialPaths = new Set<string>();
+            for (let path of initialTriggers?.values ?? []) {
+                fullInitialPaths.add(path);
+            }
+
+            // initialTriggers.parentPaths => triggerPaths
+            for (let parentPath of initialTriggers.parentPaths) {
+                let valuePaths = authorityStorage.getPathsFromParent(parentPath);
+                for (let valuePath of valuePaths || []) {
+                    fullInitialPaths.add(valuePath);
+                }
+            }
+
+            let fullInitialPathsArr = Array.from(fullInitialPaths);
+            fullInitialPathsArr = fullInitialPathsArr.filter(x => !PathRouter.isSelfAuthority(x));
+
+            let valueByPath = keyByArray(notOurValues, x => x.path);
+            for (let path of fullInitialPathsArr) {
+                let newValues = valueByPath.get(path) ?? [];
+                let prevValues = authorityStorage.getValuePlusHistory(path);
+
+                newValues.sort((a, b) => compareTime(a.time, b.time));
+
+                for (let value of prevValues) {
+                    let newIndex = binarySearchIndex(newValues.length, i => compareTime(newValues[i].time, value.time));
+                    if (newIndex < 0) {
+                        let newValue = newValues.at(-1);
+                        console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            newTimeId: newValue?.time.time,
+                        });
+                        ourValues.push({ ...value, valid: false, });
+                    }
+                }
+            }
+        }
+
+
+
+        let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
+        authorityStorage.ingestValues(ourValues, { doNotArchive });
+
+        // NOTE: Watch value locks, will internally watch remote locks if necessary. 
+        lockWatcher2.watchValueLocks(ourValues, now);
+
+        // NOTE: Initial values always get put into computeValidStates, which then ingests, but does not compute values we are not the authority for.
+        let valuesChanged = new Set<PathValue>(pathValues);
+        let dependenciesChanged = new Set<PathValue>(ourValues);
+        let alreadyTriggered = new Set<PathValue>(dependenciesChanged);
+
+        let loopCount = 0;
+
+        while (dependenciesChanged.size > 0) {
+            loopCount++;
+            if (loopCount > 3000) {
+                console.error("Too many loops in valid state loop. aborting.", {
+                    loopCount,
+                    changedCount: dependenciesChanged.size,
+                    totalTriggered: alreadyTriggered.size,
+                    exampleChanged: Array.from(dependenciesChanged)[0].path,
+                });
+                break;
+            }
+            // Just because the dependencies change doesn't mean the valid state has changed. We actually have to compute to see if it has changed. Of course, if we're not the authority, we won't compute the valid site at all. It'll just ignore it, which is good. If we're not the authority, we don't want to be doing recursive computation. Because we prepopulate it, the values change with all of the input values, we'll still get a trigger. We just won't get a nested trigger.
+            let validStateChanged = validStateComputer.computeValidStates(
+                Array.from(dependenciesChanged),
+                now,
+                initialPrevValidStates,
+            ).changed;
+            initialPrevValidStates = undefined;
+            dependenciesChanged.clear();
+
+            for (let validStateChange of validStateChanged) {
+                valuesChanged.add(validStateChange);
+                // IMPORTANT! The consequence of this is that if someone predicts a value, sends it to us, it's not valid, we will see it's not valid, and we will tell all of the nodes that are watching that path about all of the values on that path. This is inefficient. However, rejected values should be quite rare, so it shouldn't be too inefficient. 
+                if (!validStateChange.valid) {
+                    // If it's not valid, we need to send the next available valid value. But we haven't done all the fullness computation yet, so the current next available valid value might be recomputed, it might be invalid. So we'll just do the initial trigger. It's not going to be that expensive as the GCing should mean that most paths have a small history, and rejections should be quite rare. 
+                    initialTriggers.values.add(validStateChange.path);
+                }
+            }
+            // And then for everything that had a valid state change, go check if it has a dependency that was watching it.
+            for (let value of validStateChanged) {
+                let watchers = lockWatcher2.getValuePathWatchers(value);
+                for (let watcher of watchers) {
+                    if (alreadyTriggered.has(watcher)) continue;
+                    dependenciesChanged.add(watcher);
+                    alreadyTriggered.add(watcher);
+                }
+            }
+        }
+
+        pathWatcher.triggerValuesChanged({
+            valuesChanged,
+            initialTriggers
+        });
+    }
+
+    private isLockValid(lock: ReadLock): boolean {
+        let value = authorityStorage.getValueAtTime(lock.path, lock.endTime);
+        if (!value) {
+            // If not synced, we don't want to reject it yet
+            return !authorityStorage.isSynced(lock.path);
+        }
+        return !!value.valid;
+    }
+    private isLockContentionFree(lock: ReadLock): PathValue | undefined {
+        // sorted by -time (newest are first)
+        let history = authorityStorage.getValuePlusHistory(lock.path);
+
+        let index = binarySearchIndex(history.length, i => compareTime(lock.endTime, history[i].time));
+        if (index < 0) index = ~index;
+        else index++;
+        for (let i = index; i < history.length; i++) {
+            let value = history[i];
+            if (!value.valid) continue;
+            // It's fine, they have equivalent values. It was probably gced by the writer, which is fine.
+            if (value.isTransparent && lock.readIsTransparent) continue;
+            // If value.time is within our range, there's contention!
+            if (compareTime(lock.startTime, value.time) < 0 && compareTime(value.time, lock.endTime) < 0) {
+                return value;
+            }
+        }
+        return undefined;
+    }
+
+    @measureFnc
+    public computeValidStates(
+        valuePaths: PathValue[],
+        now: number,
+        prevValidStates = this.capturePrevValidStates(valuePaths)
+    ): { changed: PathValue[] } {
+        let changed: PathValue[] = [];
+
+        let lockGroups = byLockGroup(valuePaths);
+
+        for (let [locks, valueGroup] of lockGroups) {
+            if (valueGroup.length === 0) continue;
+
+            let valid = true;
+            for (let lock of locks) {
+                if (!this.isLockValid(lock)) {
+                    onPathInteracted(lock.path, 2);
+                    valid = false;
+                    this.logLockInvalid(valueGroup, lock, now);
+                    break;
+                }
+                let contention = this.isLockContentionFree(lock);
+                if (contention) {
+                    onPathInteracted(lock.path, 2);
+                    valid = false;
+                    this.logLockContention(valueGroup, lock, contention, now);
+                    break;
+                }
+            }
+            for (let pathValue of valueGroup) {
+                let prevValidState = prevValidStates.get(pathValue);
+                // IMPORTANT! This means if it didn't previously exist and it's presently rejected, we count that as a change, as it this is going from undefined to false. This is actually very useful, as a lot of places want to know if a write is rejected, so they can display it in the UI for the developer.
+                if (valid === prevValidState) continue;
+
+                changed.push(pathValue);
+
+                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it. 
+                (pathValue as any).valid = valid;
+            }
+        }
+
+        return { changed };
+    }
+
+    private logLockInvalid(valueGroup: PathValue[], lock: ReadLock, now: number) {
+        // This is good... unless it happens A LOT. Then the app needs to change (unless it
+        //  is a game, or something with realtime competition, in which case this is probably unavoidable).
+        if (valueGroup.length > 0 && lock.endTime.version !== Number.MAX_SAFE_INTEGER && lock.startTime.version !== -2) {
+            if ((!isCoreQuiet || !isNode()) && debugRejections) {
+                let timeToReject = now - valueGroup[0].time.time;
+                let message = `!!! (VALUE REJECTED) VALUE REJECTED DUE TO USING MISSING / REJECTED READ!!!(rejected after ${timeToReject}ms at ${Date.now()})`;
+                message += `\n${debugTime(valueGroup[0].time)} (write)`;
+                message += `\n    rejected as the server could not find: `;
+                if (lock.readIsTransparent) {
+                    message += `\n${debugTime(lock.startTime)} to ${debugTime(lock.endTime)} ${getPathFromStr(lock.path).join(".")} `;
+                } else {
+                    message += `\n${debugTime(lock.startTime)} ${getPathFromStr(lock.path).join(".")} `;
+                }
+                if (lock.readIsTransparent) {
+                    message += `\n    (read was undefined, so presumably a value exists which the writer missed)`;
+                }
+                message += `\nFull list of writes rejected: `;
+                for (let pathValue of valueGroup) {
+                    message += `\n${debugPathValue(pathValue)}`;
+                }
+                console.error(message);
+            }
+
+            if (isDiskAudit()) {
+                for (let pathValue of valueGroup) {
+                    auditLog("VALUE REJECTED, we did not see a required value (we are missing their value, or their value was rejected)", {
+                        path: pathValue.path,
+                        timeId: pathValue.time.time,
+                        missingPath: lock.path,
+                        missingPathTimeId: lock.startTime.time,
+                        transparent: lock.readIsTransparent,
+                    });
+                }
+            }
+        }
+    }
+    private logLockContention(valueGroup: PathValue[], lock: ReadLock, conflictingValue: PathValue, now: number) {
+
+        // This special version indicates clientside prediction (which SHOULD be rejected).
+        if (lock.endTime.version !== Number.MAX_SAFE_INTEGER) {
+
+            let changed = valueGroup.filter(x => x.valid);
+            if (changed.length > 0) {
+                if ((!isCoreQuiet || !isNode())
+                    || debugRejections
+                ) {
+                    let timeToReject = now - changed[0].time.time;
+                    let redMessage = `!!! (VALUE REJECTED) LOCK CONTENTION FIXED VIA REJECTION OF VALUE!!!(rejected after ${timeToReject}ms)`;
+                    for (let pathValue of changed) {
+                        redMessage += `\n${debugPathValue(pathValue)}`;
+                    }
+                    redMessage += `\n    (write rejected due to original read not noticing value at: ${lock.path})`;
+                    redMessage += `\n    (original read from: ${debugTime(lock.startTime)}`;
+                    redMessage += `\n    (at time: ${debugTime(lock.endTime)}`;
+                    redMessage += `\n    (current time: ${Date.now()})`;
+                    redMessage += `\n    (conflict write at: ${debugTime(conflictingValue.time)}`;
+                    console.error(redMessage);
+                }
+                if (isDiskAudit()) {
+                    for (let pathValue of changed) {
+                        auditLog("VALUE REJECTED, remote write did not see a conflicting value (remote is missing a value)", {
+                            lock,
+                            path: pathValue.path,
+                            timeId: pathValue.time.time,
+                            lockedPath: lock.path,
+                            transparent: lock.readIsTransparent,
+                            missingPath: lock.path,
+                            conflictTimeId0: conflictingValue.time.time,
+                            conflictingWrites: conflictingValue,
+                            hadTimeId: lock.startTime.time,
+                        });
+                    }
+                }
+            }
+        }
+    }
+}
+export const validStateComputer = new ValidStateComputer();

package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts

@@ -149,6 +149,6 @@ export interface ArchiveLocker {
      *      this will probably work. AND if you are reverting to an old snapshot, something that will
      *      probably work is better than a data state which is definitely broken.
      */
-    unsafeSetFiles(files: string[]): Promise<void>;
+    unsafeSetFiles(files: string[], config?: { hackNoDelete?: boolean }): Promise<void>;
     unsafeGetFileLocation(file: string): Promise<"live" | "zombie" | "recycled" | "missing">;
 }

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -15,6 +15,12 @@ import { getNodeId } from "socket-function/src/nodeCache";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
 import { parsePath, toFileNameKVP, parseFileNameKVP } from "../../misc";
 import { logDisk } from "../../diagnostics/logs/diskLogger";
+import { retryFunctional, runInParallel } from "socket-function/src/batching";
+
+/*
+    Data starts a .data files, from servers creating files.
+    Then we create .locked files, and .transactions, And when we apply these transactions, it will create .confirm files which make the .locked files count.
+*/
 
 /** Clean up old files after a while */
 const DEAD_CREATE_THRESHOLD = timeInHour * 12;
@@ -90,60 +96,83 @@ export function createArchiveLocker2(config: {
             }
         },
     };
-    async function unsafeSetFiles(files: string[]): Promise<void> {
+    async function unsafeSetFiles(files: string[], config?: { hackNoDelete?: boolean }): Promise<void> {
         let valuePaths = new Set((await archiveValues.findInfo("")).map(x => x.path));
         let correctConfirms = new Set<string>();
-        let correctFiles = new Set<string>();
+        let correctFiles = new Set<string>(files);
 
         async function deleteAllOtherFiles() {
+            if (config?.hackNoDelete) return;
             // Delete all old confirms / transactions / etc
             let allConfirms = (await archiveLocks.findInfo("")).map(x => x.path);
-            await Promise.all(allConfirms.map(async confirm => {
+            console.log(`Found ${allConfirms.length} confirms to delete`);
+            let deleteCount = 0;
+            async function deleteConfirm(confirm: string) {
+                deleteCount++;
                 if (correctConfirms.has(confirm)) return;
                 await storage.deleteKey(confirm);
-            }));
+                console.log(`Deleted confirm ${confirm}, ${deleteCount}/${allConfirms.length}`);
+            }
+            let deleteConfirmParallel = runInParallel({ parallelCount: 8 }, deleteConfirm);
+            await Promise.all(allConfirms.map(deleteConfirmParallel));
+            console.log(`Deleted ${allConfirms.length} confirms`);
 
             valuePaths = new Set((await archiveValues.findInfo("")).map(x => x.path));
+            console.log(`Found ${valuePaths.size} files to delete`);
             // Delete all files we didn't just set (move to recycle bin)
-            await Promise.all(Array.from(valuePaths).map(async file => {
+            let moveCount = 0;
+            async function moveFile(file: string) {
+                moveCount++;
                 if (correctFiles.has(file)) return;
+                let info = await archiveValues.getInfo(file);
+                console.log(`Moving file ${file}, ${moveCount}/${valuePaths.size}, ${formatNumber(info?.size || 0)} bytes`);
                 await archiveValues.move({
                     path: file,
                     targetPath: file,
                     target: archiveRecycleBin,
                 });
-            }));
+                console.log(`Moved file ${file}, ${moveCount}/${valuePaths.size}`);
+            }
+            let moveFileParallel = runInParallel({ parallelCount: 8 }, moveFile);
+            await Promise.all(Array.from(valuePaths).map(moveFileParallel));
+            console.log(`Deleted ${valuePaths.size} files`);
         }
 
-        console.log(magenta(`Deleting all other files`));
+        console.log(magenta(`Deleting all other files (this may take a while)`));
         // Deleting early deletes any transactions, which might be reapplied due to our non-transaction changes
         await deleteAllOtherFiles();
 
-        // Undelete, by retrieving all files from the recycle bin
-        let promises: Promise<void>[] = [];
-        for (let file of files) {
-            if (valuePaths.has(file)) {
-                correctFiles.add(file);
+        let restoreCount = 0;
+        async function restoreFile(file: string): Promise<void> {
+            if (await archiveValues.get(file)) {
+                console.log(`File ${file} is already in the archive values, skipping, ${restoreCount++}/${files.length}`);
+                return;
             }
             let isInRecycleBin = !!(await archiveRecycleBin.getInfo(file));
             if (isInRecycleBin) {
-                console.log(`Restoring ${file}`);
-                promises.push(archiveRecycleBin.move({
+                console.log(`Restoring ${file}, ${restoreCount++}/${files.length}`);
+                await archiveRecycleBin.move({
                     path: file,
                     targetPath: file,
                     target: archiveValues,
-                }));
-                correctFiles.add(file);
+                });
+            } else {
+                console.log(`File ${file} is not in the recycle bin, skipping, ${restoreCount++}/${files.length}`);
             }
         }
-        await Promise.all(promises);
+        const restoreFileParallel = runInParallel({ parallelCount: 16 }, restoreFile);
+        await Promise.all(files.map(restoreFileParallel));
 
         console.log(magenta(`Creating confirms`));
+        let createCount = 0;
         // Confirm all files
-        await Promise.all(Array.from(correctFiles).map(async file => {
-            let confirm = await locker.createConfirm(file);
+        async function createConfirm(file: string): Promise<void> {
+            let confirm = await locker.createConfirm(file, correctFiles.size - createCount);
             correctConfirms.add(confirm);
-        }));
+            console.log(`Created confirm ${confirm}, ${createCount++}/${correctFiles.size}`);
+        }
+        let createConfirmParallel = runInParallel({ parallelCount: 4 }, retryFunctional(createConfirm));
+        await Promise.all(Array.from(correctFiles).map(createConfirmParallel));
 
         console.log(magenta(`Deleting all other files again`));
         // Delete again, in case anything else was created while we were restoring. Not great, but... should
@@ -184,6 +213,7 @@ export function createArchiveLocker2(config: {
             ) => Promise<ArchiveTransaction[]>
         ): Promise<"accepted" | "rejected"> {
             let files = await locker.getFiles();
+            await saveSnapshot({ files: files.map(a => a.file) });
             let readFiles = async (files: FileInfo[]) => {
                 let pendingFiles = files.slice();
                 let readResults = new Map<string, Buffer | undefined>();
@@ -231,7 +261,6 @@ export function createArchiveLocker2(config: {
                         let unhandled: never = type;
                     }
                 }
-                logErrors(saveSnapshot({ files: Array.from(newFiles) }));
             }
             return status;
         }
@@ -259,7 +288,7 @@ type StorageType = {
      *          all writes should now be readable.
      */
 };
-type Transaction = {
+export type Transaction = {
     ops: ({
         type: "create";
         key: string;
@@ -307,15 +336,16 @@ class TransactionLocker {
         let { dir, name } = parsePath(key);
         return `${dir}confirm_${name}.confirm`;
     }
-    public async createConfirm(key: string) {
+    public async createConfirm(key: string, countLeft: number) {
         let path = this.getConfirmKey(key);
-        console.info("Creating confirmation for ${key}");
-        await this.storage.setValue(path, Buffer.from(""));
+        console.info(`Creating confirmation for ${key}, ${countLeft} left`);
+        if (!await this.storage.getValue(path)) {
+            await this.storage.setValue(path, Buffer.from(""));
+        }
         return path;
     }
-    private async deleteDataFile(key: string, reason: string): Promise<void> {
-        console.log(red(`Deleting data file ${key}, because ${reason}`));
-        //await this.storage.setValue(key + ".reason", Buffer.from(reason));
+    private async deleteDataFile(key: string, reason: string, countLeft: number): Promise<void> {
+        console.log(red(`Deleting data file ${key}, because ${reason}, ${countLeft} left`));
         // Delete file, and confirmation as well
         await this.storage.deleteKey(key);
         await this.storage.deleteKey(this.getConfirmKey(key));
@@ -472,13 +502,22 @@ class TransactionLocker {
 
             let existingFiles = new Map(files.map(a => [a.file, a]));
 
+            let uncomfirmedCount = 0;
+
             let currentDataFiles = new Map<string, FileInfo>();
+            let countByExtension = new Map<string, { value: number }>();
             for (let file of files) {
+                let extension = file.file.split(".").pop() || "";
+                let existing = countByExtension.get(extension);
+                if (!existing) {
+                    existing = { value: 0 };
+                    countByExtension.set(extension, existing);
+                }
+                existing.value++;
                 if (!(
                     file.file.endsWith(".locked")
                     || file.file.endsWith(".confirm")
                     || file.file.endsWith(".transaction")
-                    || file.file.endsWith(".reason")
                 )) {
                     currentDataFiles.set(file.file, file);
                     continue;
@@ -488,14 +527,17 @@ class TransactionLocker {
                 let confirmFile = existingFiles.get(confirmKey);
                 if (confirmFile) {
                     currentDataFiles.set(file.file, file);
+                } else {
+                    uncomfirmedCount++;
                 }
             }
-
-            console.info("Read archive state", {
+            console.log("Read archive state", {
                 rawFilesCount: files.length,
                 confirmedCount: currentDataFiles.size,
-                rawFiles: files.map(a => a.file),
-                confirmedFiles: Array.from(currentDataFiles.values()).map(a => a.file),
+                uncomfirmedCount,
+                countByExtention: Array.from(countByExtension.entries()).map(([extension, value]) => `${extension}: ${value.value}`).join(" | "),
+                //rawFiles: files.map(a => a.file),
+                //confirmedFiles: Array.from(currentDataFiles.values()).map(a => a.file),
             });
 
             return {
@@ -586,9 +628,9 @@ class TransactionLocker {
                 let op = opsRemaining.pop();
                 if (!op) return;
                 if (op.type === "create") {
-                    await this.createConfirm(op.key);
+                    await this.createConfirm(op.key, opsRemaining.length);
                 } else if (op.type === "delete") {
-                    await this.deleteDataFile(op.key, `transaction (${getOwnNodeId()})`);
+                    await this.deleteDataFile(op.key, `transaction (${getOwnNodeId()})`, opsRemaining.length);
                 } else {
                     let unhandled: never = op;
                     throw new Error(`Unhandled type: ${unhandled}`);
@@ -706,8 +748,9 @@ class TransactionLocker {
                         console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`), { files: unconfirmedOldFiles2.map(x => x.file) });
                         logNodeStats(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
                         // At the point the file was very old when we started reading, not part of the active transaction.
-                        for (let file of unconfirmedOldFiles2) {
-                            await this.deleteDataFile(file.file, `old unconfirmed file`);
+                        for (let i = 0; i < unconfirmedOldFiles2.length; i++) {
+                            let file = unconfirmedOldFiles2[i];
+                            await this.deleteDataFile(file.file, `old unconfirmed file`, unconfirmedOldFiles2.length - i);
                         }
                     } else {
                         console.warn(`Almost deleted ${unconfirmedOldFiles.length} very old unconfirmed files. This is bad, did we miss their confirmations that first time? If we missed them twice in a row, we might literally delete the database, and need to enter recovery mode to fix it...`, { files: unconfirmedOldFiles });
@@ -740,13 +783,6 @@ class TransactionLocker {
                 }
             }
 
-            // Delete some debug files
-            {
-                for (let file of dataState.rawDataFiles.filter(x => x.file.endsWith(".reason"))) {
-                    await this.storage.deleteKey(file.file);
-                }
-            }
-
             return {
                 dataFiles: Array.from(dataState.confirmedDataFiles),
                 rawDataFiles: dataState.rawDataFiles,

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -4,7 +4,7 @@ import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDisco
 import { pathValueArchives } from "../pathValueArchives";
 import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";
 import { green, magenta } from "socket-function/src/formatting/logColors";
-import { devDebugbreak } from "../../config";
+import { devDebugbreak, isPublic } from "../../config";
 import { sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 
@@ -56,6 +56,7 @@ export async function saveSnapshot(config: {
     let { files } = config;
     let overview = getSnapshotOverview(files);
     await snapshots().set(overview.file, Buffer.from(files.join("\n")));
+    return overview;
 }
 function overviewToFileName(overview: Omit<ArchiveSnapshotOverview, "file">): string {
     return Object.entries({
@@ -152,10 +153,11 @@ async function getSnapshotBase(snapshotFile: string | "live") {
 //      although you will still have to take most of the servers down to get back
 //      to a consistent server state.
 export async function loadSnapshot(config: {
-    overview: ArchiveSnapshotOverview;
-    audit?: (nodeId: string) => Promise<void>;
+    overview: { file: string; };
+    noExit?: boolean;
+    hackNoDelete?: boolean;
 }): Promise<void> {
-    let { overview, audit } = config;
+    let { overview } = config;
     console.log(magenta(`Loading snapshot: ${overview.file}`));
     let locker = await pathValueArchives.getArchiveLocker();
 
@@ -166,20 +168,15 @@ export async function loadSnapshot(config: {
     let files = snapshotData.toString().split("\n");
 
     console.log(magenta(`Unsafe setting snapshot files: ${overview.file}`));
-    await locker.unsafeSetFiles(files);
+    await locker.unsafeSetFiles(files, { hackNoDelete: config.hackNoDelete });
     console.log(magenta(`Finished unsafe setting snapshot files: ${overview.file}`));
 
-    let allNodes = await getAllNodeIds();
-
-    // Logging the errors, not ignoring them
-    if (audit) {
-        await timeoutToUndefinedSilent(10_000, Promise.all(allNodes.map(async nodeId => {
-            if (!audit) return;
-            await audit(nodeId);
-        })));
-    }
-
     for (let i = 0; i < 10; i++) {
         console.log(green(`!!!Finished loading snapshot!!!`));
     }
-}
+
+    if (!config.noExit && !isPublic()) {
+        // Exit so we know when this is done. Really, all the servers need to be restarted after we load the snapshot, anyway. 
+        process.exit();
+    }
+}

package/src/0-path-value-core/auditLogs.ts

@@ -47,7 +47,9 @@ export function getLogHistoryEquals(value: string) {
 (globalThis as any).getLogHistoryEquals = getLogHistoryEquals;
 (globalThis as any).getPathAuditLogs = getLogHistoryEquals;
 
+// NOTE: It's good practice to check isDebugLogEnabled before calling this, as it's expensive to create an object if we're just going to throw it away. 
 export function auditLog(type: string, values: { [key: string]: unknown }) {
+    if (!ENABLED_LOGGING) return;
     debugLogFnc(type, values);
 }
 let debugLogFnc = debugLogBase;