querysub 0.2.0

package/src/5-diagnostics/diskValueAudit.ts

@@ -0,0 +1,216 @@
+import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { AuthorityPath, isInAuthority, nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { ARCHIVE_FLUSH_LIMIT, MAX_ARCHIVE_PROPAGATION_DELAY, PathValue, authorityStorage, epochTime, pathValueArchives, pathWatcher } from "../0-path-value-core/pathValueCore";
+import { isClient } from "../config2";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { PathValueArchives } from "../0-path-value-core/pathValueArchives";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { green } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { isDevDebugbreak } from "../config";
+import { DebugLog, getLogHistoryEquals } from "../0-path-value-core/debugLogs";
+import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { sort } from "socket-function/src/misc";
+
+const AUDIT_INTERVAL = ARCHIVE_FLUSH_LIMIT;
+const AUDIT_THRESHOLD = ARCHIVE_FLUSH_LIMIT;
+
+export async function startDiskAuditLoop() {
+    if (isClient()) return;
+
+    runInfinitePoll(AUDIT_INTERVAL, diskAuditNow);
+}
+
+(globalThis as any).diskAuditNow = diskAuditNow;
+export async function diskAuditNow() {
+    let selfAuthorities = nodePathAuthority.getSelfAuthorities();
+    for (let authority of selfAuthorities) {
+        await auditAuthority(authority);
+    }
+}
+
+const auditAuthority = runInSerial(async function auditAuthority(authority: AuthorityPath) {
+    let threshold = Date.now() - AUDIT_THRESHOLD;
+    let time = Date.now();
+    await checkAuthority(authority, threshold);
+
+    console.log(green(`Checked authority ${pathValueAuthority2.getArchiveDirectory(authority)} in ${formatTime(Date.now() - time)}`));
+});
+
+async function getAuthorityBuffers(authority: AuthorityPath, threshold: number) {
+    let bufferCache = new Map<string, Buffer>();
+    let loopCount = 0;
+    let reread = true;
+
+    while (reread) {
+        reread = false;
+        let valuePaths = await pathValueArchives.getValuePaths(authority);
+        valuePaths = valuePaths.filter(path => {
+            let obj = pathValueArchives.decodeDataPath(path);
+            if (obj.minTime && obj.minTime > threshold) return false;
+            return true;
+        });
+        for (let valuePath of valuePaths) {
+            if (bufferCache.has(valuePath)) continue;
+            let rawValue = (await pathValueArchives.getRawValueFiles([valuePath]))[0];
+            if (!rawValue) {
+                loopCount++;
+                if (loopCount > 10) {
+                    console.error(`Looped too many times trying to get an consistent state. It looks as if pathValueArchives.getValuePaths(${pathValueAuthority2.getArchiveDirectory(authority)}) is returning an invalid/missing path, ${valuePath}`);
+                    return undefined;
+                }
+                reread = true;
+                break;
+            }
+            bufferCache.set(valuePath, rawValue);
+        }
+    }
+    return Array.from(bufferCache.entries());
+}
+
+// NOTE: We only check for { path, Time }, because presumably the value won't become corrupted. HOWEVER,
+//      getting out of sync with the disk is not only possible, but likely, as there are probably hundreds
+//      places in which this can happen, due to complex race conditions.
+async function checkAuthority(authority: AuthorityPath, threshold: number) {
+    let buffers = await getAuthorityBuffers(authority, threshold);
+    if (!buffers) return;
+
+    let diskValues = new Map<string, PathValue>();
+    for (let [path, data] of buffers) {
+        let values = await PathValueArchives.loadValuesFromBuffer({
+            path, data,
+        });
+        let countSinceDelay = 0;
+        for (let diskValue of values.values) {
+            countSinceDelay++;
+            if (countSinceDelay > 10_000) {
+                countSinceDelay = 0;
+                await delay("paintLoop");
+            }
+
+            let prev = diskValues.get(diskValue.path);
+            if (prev && prev.time.time > diskValue.time.time) continue;
+            diskValues.set(diskValue.path, diskValue);
+        }
+    }
+
+    let changedValues = new Set<PathValue>();
+    let removedValues = new Set<PathValue>();
+
+    let countSinceDelay = 0;
+    for (let [path, diskValue] of diskValues) {
+        if (countSinceDelay > 10_000) {
+            countSinceDelay = 0;
+            await delay("paintLoop");
+        }
+
+        if (!isInAuthority(authority, diskValue.path)) continue;
+        if (diskValue.time.time > threshold) continue;
+
+        // Ignore cases where deletes happened too recently, as they won't be reflected
+        //  on the disk, so we should EXPECT a difference.
+        let lastChangedTime = authorityStorage.getLastChangedTime(diskValue.path);
+        if (lastChangedTime && lastChangedTime.time > threshold) continue;
+
+        let memoryValue = authorityStorage.getValueAtTime(diskValue.path);
+        let values = authorityStorage.getValuePlusHistory(diskValue.path);
+        let time = Date.now();
+        if (pathValueSerializer.compareValuePaths(memoryValue, diskValue)) {
+            // IF we hit this, it likely means our auditing is wrong, and we are
+            //  creating spurious audit failures.
+            // OTHERWISE, it means we have a fundamental issue how we handle ValuePaths, because
+            //      locally we aren't even running multiple servers, so we should NEVER get
+            //      corrupted / out of sync data!
+            if (isDevDebugbreak()) {
+                let searchValue = path;
+                let logs: DebugLog[] = getLogHistoryEquals(path);
+                let atTime = Date.now();
+                console.error(`Audit mismatch for ${path} from detected at ${atTime}`);
+                console.log(`Disk value:`, pathValueSerializer.getPathValue(diskValue));
+                console.log(`Memory value:`, pathValueSerializer.getPathValue(memoryValue));
+                console.log(`Disk time:`, diskValue.time.time);
+                console.log(`Memory time:`, memoryValue?.time.time);
+                console.log();
+                sort(logs, x => x.time);
+                let now = Date.now();
+                for (let log of logs) {
+                    console.log(`${log.type.padEnd(20, " ")}`, now - log.time, log.values, log.time);
+                }
+                debugbreak(2);
+                debugger;
+                console.log(path);
+                console.log(logs.length);
+                console.log(atTime);
+            }
+            changedValues.add(diskValue);
+            console.log(values, time, memoryValue, diskValue);
+        }
+    }
+
+    // OH! And then we also have to check for removed values! (Really the most important part,
+    //      as this happens normally when we GC).
+    for (let [memoryPath, value] of authorityStorage.__auditValues()) {
+        if (countSinceDelay > 10_000) {
+            countSinceDelay = 0;
+            await delay("paintLoop");
+        }
+        if (value.length === 0) continue;
+        if (value[0].time.time > threshold) continue;
+        let latestValue = value[0];
+        if (!value[0].valid) {
+            let latestValidValue = value.find(v => v.valid);
+            if (!latestValidValue) continue;
+            latestValue = latestValidValue;
+        }
+        if (latestValue.time.time > threshold) continue;
+        // If the value is known to be GCable... we then it not existing is equivalent. ALSO,
+        //  it should have really been removed from memory already? But that's the audit code's problem.
+        if (latestValue.canGCValue) continue;
+        let diskValue = diskValues.get(memoryPath);
+        // There is a value in memory, but not on disk, so we must remove it from memory.
+        if (!diskValue || diskValue.canGCValue) {
+            removedValues.add({
+                path: memoryPath,
+                time: epochTime,
+                value: undefined,
+                valid: true,
+                canGCValue: true,
+                isTransparent: true,
+                lockCount: 0,
+                locks: [],
+            });
+        }
+    }
+    if (changedValues.size > 0) {
+        let firstValue = changedValues.values().next().value as PathValue;
+        console.error(`Value mismatch between disk and memory for ${formatNumber(changedValues.size)} values. Ex: ${firstValue.path}`);
+    }
+    if (removedValues.size > 0) {
+        console.info(green(`Removing ${formatNumber(removedValues.size)} paths from memory which have been GCed on the disk.`));
+    }
+    if (changedValues.size > 0 || removedValues.size > 0) {
+        let allValues = new Set([...changedValues, ...removedValues]);
+        // Check again, as our lag waiting means there is a race condition between finding a removed
+        //  path, and the user touching it again!
+        for (let valuePath of Array.from(allValues)) {
+            let lastChangedTime = authorityStorage.getLastChangedTime(valuePath.path);
+            if (lastChangedTime && lastChangedTime.time > threshold) {
+                allValues.delete(valuePath);
+            } else {
+                authorityStorage.resetForInitialTrigger(valuePath.path);
+            }
+        }
+        authorityStorage.ingestValues(Array.from(allValues), undefined, undefined);
+
+        if (allValues.size > 0) {
+            // NOTE: By triggering all of these as initial syncs... it SHOULD result in us clobbering the old
+            //  values. AND, we shouldn't have to worry about race conditions, because this is synchronous,
+            //  and so any new values will happen and be queued after us.
+            //  - Except predictions might be clobbered... but that's fine, they don't really matter.
+            pathWatcher.triggerValuesChanged(allValues, undefined, {
+                parentPaths: new Set(),
+                values: allValues,
+            });
+        }
+    }
+}

package/src/5-diagnostics/memoryValueAudit.ts

@@ -0,0 +1,442 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { PathValueController } from "../0-path-value-core/PathValueController";
+import { PathValue, authorityStorage, pathWatcher } from "../0-path-value-core/pathValueCore";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { isNode, list, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { getSingleSizeEstimate } from "./shared";
+import { devDebugbreak, isDevDebugbreak } from "../config";
+import { nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
+import { isClient } from "../config2";
+import { StringSerialize } from "../-h-path-value-serialize/stringSerializer";
+import { delay, runInfinitePoll } from "socket-function/src/batching";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import debugbreak from "debugbreak";
+import { DebugLog, DebugLogController, getLogHistoryEquals, getLogHistoryIncludes } from "../0-path-value-core/debugLogs";
+import { getParentPathStr } from "../path";
+import { diskAuditNow } from "./diskValueAudit";
+import { yellow, green } from "socket-function/src/formatting/logColors";
+import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
+
+const AUDIT_COUNT = 100;
+const AUDIT_MAX_BYTES_PER_SECOND = isNode() ? 1024 * 1024 : 1024 * 10;
+const AUDIT_INTERVAL = timeInMinute * 5;
+const MAX_RESYNC_INTERVAL = timeInMinute * 60;
+// NOTE: This is quite short, because it isn't really the network delay, but more our internal delay
+// NOTE: Longer now, because I think we were running into lag issues
+const MAX_ASSUMED_PROPAGATION_DELAY = 1000 * 30;
+const MAX_AUDIT_GROUP_SIZE = 1000 * 10;
+
+export async function startMemoryAuditLoop() {
+    // NOTE: For now... there are security issues with allowing clients to audit paths.
+    //  We COULD update the algorithm to work with Querysub directly, but... this
+    //  shouldn't be needed. Clients SHOULDN'T get out of sync (they are only talking
+    //  with one service, Querysub), and if they do, the user can refresh to fix it.
+    //  (Where as if a Querysub itself is out of sync, it need to fix it, otherwise
+    //  every single client will get the wrong values).
+    if (isClient()) return;
+
+    //  Re-enable this
+    runInfinitePoll(AUDIT_INTERVAL, async () => {
+        await auditSomeOfOurPaths();
+        await auditSomeRemotePaths();
+    });
+}
+
+async function fullMemoryAudit() {
+    console.log(yellow(`Starting full memory audit`));
+    {
+        let values = authorityStorage.__auditValues();
+        let paths = Array.from(values.keys());
+        await auditPaths({ paths });
+    }
+
+    let remoteNodeIds = remoteWatcher.getAllRemoteWatchedNodeIds();
+    for (let remoteNodeId of remoteNodeIds) {
+        console.log(yellow(`    Auditing all paths from ${remoteNodeId}`));
+        let parentWatches = pathWatcher.getAllParentWatches();
+        // Get ALL paths under parent paths
+        //  (hopefully this doesn't crash the server... it is dev triggered, so...
+        //      they should know what they are doing)
+        let randomPaths = await ValueAuditController.nodes[remoteNodeId].getRandomPaths({
+            parentWatches: parentWatches,
+            count: Number.MAX_SAFE_INTEGER
+        });
+        await auditSpecificPaths({ paths: randomPaths, authorityNodeId: remoteNodeId });
+    }
+    console.log(green(`Finished full memory audit`));
+}
+
+async function auditSomeOfOurPaths() {
+    let ourPaths = await getRandomPaths({ count: AUDIT_COUNT });
+    await auditPaths({ paths: ourPaths });
+}
+// NOTE: Exposed for testing, but... might be useful in the future?
+(globalThis as any).auditPaths = auditPaths;
+async function auditPaths(config: {
+    paths: string[];
+    forceAudit?: boolean;
+}) {
+    let { paths } = config;
+    for (let i = 0; i < paths.length; i += MAX_AUDIT_GROUP_SIZE) {
+        await auditPathsBase({
+            paths: paths.slice(i, i + MAX_AUDIT_GROUP_SIZE),
+            forceAudit: config.forceAudit,
+        });
+        await delay("paintLoop");
+    }
+}
+async function auditPathsBase(config: {
+    paths: string[];
+    forceAudit?: boolean;
+}) {
+    const { paths } = config;
+    // Split paths by nodeId
+    let pathByAuthority = new Map<string, Set<string>>();
+
+    for (let path of paths) {
+        let remoteNodeId = remoteWatcher.getExistingWatchRemoteNodeId(path);
+        if (!remoteNodeId) continue;
+        if (isOwnNodeId(remoteNodeId)) continue;
+        let paths = pathByAuthority.get(remoteNodeId);
+        if (!paths) {
+            paths = new Set();
+            pathByAuthority.set(remoteNodeId, paths);
+        }
+        paths.add(path);
+    }
+    for (let [authorityNodeId, paths] of pathByAuthority) {
+        await auditSpecificPaths({
+            paths: Array.from(paths),
+            authorityNodeId,
+            forceAudit: config.forceAudit,
+        });
+    }
+}
+// Needed to find if we are missing any paths
+async function auditSomeRemotePaths() {
+    let remoteNodeIds = remoteWatcher.getAllRemoteWatchedNodeIds();
+    if (remoteNodeIds.length === 0) return [];
+    // Pick a random one
+    let remoteNodeId = remoteNodeIds[Math.floor(Math.random() * remoteNodeIds.length)];
+    if (isOwnNodeId(remoteNodeId)) return;
+    // NOTE: The only remote paths we will be missing are from parent watches, so...
+    //  only check random parent paths.
+    let parentWatches = pathWatcher.getAllParentWatches();
+    let randomParentWatches = sampleRandom({ count: AUDIT_COUNT, paths: parentWatches });
+    let randomPaths = await ValueAuditController.nodes[remoteNodeId].getRandomPaths({ parentWatches: randomParentWatches, count: AUDIT_COUNT });
+    let watched = pathWatcher.getWatchers(new Set(randomPaths.map(path => ({ path }))));
+    let watchedPaths = Array.from(new Set(Array.from(watched.values()).map(x => Array.from(x).map(x => x.path)).flat()));
+    if (watchedPaths.length > 0) {
+        await auditSpecificPaths({ paths: watchedPaths, authorityNodeId: remoteNodeId });
+    }
+}
+
+async function getRandomPaths(config: {
+    count: number;
+}): Promise<string[]> {
+    let values = authorityStorage.__auditValues();
+    let paths = Object.keys(values);
+    let superset = sampleRandom({ count: config.count * 3, paths });
+    superset = superset.filter(path => !values.get(path)?.[0]?.event);
+    return superset.slice(0, config.count);
+}
+function sampleRandom(config: {
+    count: number;
+    paths: string[];
+}): string[] {
+    let { count, paths } = config;
+    if (count > paths.length) count = paths.length;
+    if (count / paths.length < 0.3) {
+        // For few values looping is faster
+        return getLoopRandom();
+    } else {
+        // If all values have to be picking looping is too slow. Use sorting
+        return getSortedRandom();
+    }
+    function getLoopRandom() {
+        let pickedIndexes = new Set<number>();
+        while (pickedIndexes.size < count) {
+            let index = Math.floor(Math.random() * paths.length);
+            pickedIndexes.add(index);
+        }
+        return Array.from(pickedIndexes).map(i => paths[i]);
+    }
+    function getSortedRandom() {
+        let random = new Array(paths.length).map(() => Math.random());
+        let indexes = list(paths.length);
+        sort(indexes, i => random[i]);
+        return indexes.map(i => paths[i]);
+    }
+}
+
+const AUDIT_WINDOW_SIZE = 10;
+let lastAuditsSizes: {
+    count: number;
+    size: number;
+    time: number;
+}[] = [];
+
+async function auditSpecificPaths(config: {
+    paths: string[];
+    authorityNodeId: string;
+    forceAudit?: boolean;
+}) {
+    let { paths } = config;
+    for (let i = 0; i < paths.length; i += MAX_AUDIT_GROUP_SIZE) {
+        await auditSpecificPathsBase({
+            paths: paths.slice(i, i + MAX_AUDIT_GROUP_SIZE),
+            authorityNodeId: config.authorityNodeId,
+            forceAudit: config.forceAudit,
+        });
+        await delay("paintLoop");
+        if (i + MAX_AUDIT_GROUP_SIZE < paths.length) {
+            console.log(`        Audit progress: ${formatPercent((i + MAX_AUDIT_GROUP_SIZE) / paths.length)} / ${formatNumber(paths.length)} paths`);
+        }
+    }
+}
+async function auditSpecificPathsBase(config: {
+    paths: string[];
+    authorityNodeId: string;
+    forceAudit?: boolean;
+}) {
+    let { paths, authorityNodeId, forceAudit } = config;
+    paths = paths.filter(path => !nodePathAuthority.isSelfAuthority(path));
+    if (paths.length === 0) return;
+
+
+    // Limit paths based on recent audit sizes (if we are downloading too much data,
+    //      audit fewer paths, up to just auditing a single path).
+    //  Use getSingleSizeEstimate to get sizes
+    // If forceAudit, don't limit the paths
+    if (lastAuditsSizes.length > 0 && !config.forceAudit) {
+        let auditWindowTotalSize = lastAuditsSizes.map(v => v.size).reduce((a, b) => a + b, 0);
+        let auditWindowTime = Date.now() - lastAuditsSizes[0].time;
+        let auditWindowRate = auditWindowTotalSize / auditWindowTime;
+        let fractionToReduce = Math.min(1, auditWindowRate / AUDIT_MAX_BYTES_PER_SECOND);
+        let count = Math.max(1, Math.floor(paths.length * fractionToReduce));
+        paths = paths.slice(0, count);
+    }
+
+    let invalidValues = new Set<string>();
+    let remoteValues = await ValueAuditController.nodes[authorityNodeId].getPathValues({ paths });
+
+    {
+        let size = remoteValues.map(v =>
+            (v.value ? getSingleSizeEstimate(v.value) : 0)
+            + v.path.length * 2
+        ).reduce((a, b) => a + b, 0);
+        lastAuditsSizes.push({ count: paths.length, size, time: Date.now() });
+        while (lastAuditsSizes.length > AUDIT_WINDOW_SIZE) {
+            lastAuditsSizes.shift();
+        }
+    }
+
+    const THRESHOLD_TIME = Date.now() - MAX_ASSUMED_PROPAGATION_DELAY;
+    let waited = false;
+    for (let { path, value } of remoteValues) {
+        let ourValue = authorityStorage.getValueAtTime(path);
+        if (!ourValue && !value) continue;
+
+        // No point in auditing event values, they are definitely going to disappear shortly!
+        if (ourValue?.event || value?.event) continue;
+
+        // If the value JUST changed... assume we just haven't received the update yet.
+        //  - This DOES cause issues if the value is changing VERY frequently, BUT...
+        //      to know this we would have to 
+        if (ourValue && ourValue.time.time > THRESHOLD_TIME) {
+            continue;
+        }
+
+        if (!pathValueSerializer.compareValuePaths(value, ourValue)) continue;
+
+        let ourTime = ourValue ? ourValue.time.time : 0;
+        let remoteTime = value ? value.time.time : 0;
+        let oursWasNew = ourTime > THRESHOLD_TIME;
+        let theirsWasNew = remoteTime > THRESHOLD_TIME;
+
+        // Wait, and try again
+        if (!waited) {
+            waited = true;
+            await delay(MAX_ASSUMED_PROPAGATION_DELAY);
+        }
+        ourValue = authorityStorage.getValueAtTime(path);
+        value = (await ValueAuditController.nodes[authorityNodeId].getPathValue(path)).value;
+
+        if (!pathValueSerializer.compareValuePaths(value, ourValue)) continue;
+        let THRESHOLD2 = Date.now() - MAX_ASSUMED_PROPAGATION_DELAY;
+        let ourTime2 = authorityStorage.getLastChangedTime(path)?.time || 0;
+        let oursIsNew = ourTime2 > THRESHOLD2;
+        let theirTime2 = value ? value.time.time : 0;
+        let theirsIsNew = theirTime2 > THRESHOLD2;
+        if ((theirsIsNew || theirsWasNew) && (oursIsNew || oursWasNew)) {
+            // Both are updating.Which means they will probably sync. It is highly unlikely for our
+            //  value to be constantly updated, and stay wrong (and if it is,
+            //  then resyncing won't help!)
+            continue;
+        }
+        let reason = remoteWatcher.getWatchReason(path);
+        if (!reason) continue;
+
+        if (isDevDebugbreak()) {
+            const ourNodeId = getOwnNodeId();
+            let logs: DebugLog[] = [];
+            let remoteLogs: DebugLog[] = [];
+            // NOTE: Actually, even if we included it because of a parent watch, we still only
+            //      care about direct writes to this value.
+            if (reason === "parent") {
+                logs = getLogHistoryIncludes(path);
+                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
+
+                let logs2 = getLogHistoryIncludes(getParentPathStr(path));
+                let remoteLogs2 = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(getParentPathStr(path));
+                logs.push(...logs2);
+                remoteLogs.push(...remoteLogs2);
+            } else {
+                logs = getLogHistoryEquals(path);
+                remoteLogs = await DebugLogController.nodes[authorityNodeId].getLogHistoryEquals(path);
+            }
+            let atTime = Date.now();
+            console.error(`Audit mismatch for ${path} from ${authorityNodeId} detected at ${atTime}. Watching because: ${reason}`);
+            console.log(`Value Remote:`, pathValueSerializer.getPathValue(value));
+            console.log(`Value Local:`, pathValueSerializer.getPathValue(ourValue));
+            console.log(`Time Remote:`, value?.time.time);
+            console.log(`Time Local:`, ourValue?.time.time);
+            console.log(`Our node id:`, ourNodeId);
+            console.log(`Time now:`, atTime);
+            console.log();
+            let logObjs: { source: string, log: DebugLog }[] = [];
+            for (let log of logs) {
+                logObjs.push({ source: "local", log });
+            }
+            for (let log of remoteLogs) {
+                logObjs.push({ source: "remote", log });
+            }
+            sort(logObjs, x => x.log.time);
+            let now = Date.now();
+            for (let { source, log } of logObjs) {
+                console.log(`${source.padEnd(8, " ")} ${log.type.padEnd(20, " ")}`, now - log.time, log.values, log.time);
+            }
+            debugbreak(2);
+            debugger;
+            console.log(path);
+            console.log(value);
+            console.log(ourValue);
+            console.log(logs.length);
+            console.log(remoteLogs.length);
+            console.log(atTime);
+        }
+
+        invalidValues.add(path);
+    }
+    if (invalidValues.size > 0) {
+        await resyncInvalidValues({ paths: Array.from(invalidValues), authorityNodeId, fromForceAudit: forceAudit });
+    }
+}
+
+let lastResyncTime = 0;
+// NOTE: For now, we'll just resync all watches, just to be safe.
+async function resyncInvalidValues(config: {
+    paths: string[];
+    authorityNodeId: string;
+    fromForceAudit?: boolean;
+}) {
+    const { paths, authorityNodeId } = config;
+
+    if (!config.fromForceAudit) {
+        let skipResync = Date.now() - lastResyncTime < MAX_RESYNC_INTERVAL;
+        //  console.error, listing a few paths, authorityNodeId, etc
+        {
+            let samplePaths = paths.slice(0, 3);
+            console.error(`${skipResync ? "SKIPPING RESYNC DUE TO EXCESSIVE RESYNCS" : "Resyncing"} ${samplePaths.length} paths from ${authorityNodeId} due to audit mismatch. Some paths that failed are: ${JSON.stringify(samplePaths)}`);
+        }
+        if (skipResync) return;
+    }
+    lastResyncTime = Date.now();
+
+
+    // If we tell the remote server to audit from a forced audit... we might infinitely loop
+    //  back and forth with audits, which defeats the point of auditing to increase stability!
+    if (isNode()) {
+        // Tell the remote server to audit the specific paths, securely.
+        //      AND wait until it audits them.
+        await ValueAuditController.nodes[authorityNodeId].auditNow({ paths });
+    }
+
+    await remoteWatcher.refreshAllWatches(config.authorityNodeId);
+}
+
+
+class ValueAuditControllerBase {
+    public async getPathValue(path: string) {
+        let value = authorityStorage.getValueAtTime(path);
+        return { path, value };
+    }
+    public async getPathValues(config: {
+        paths: string[];
+    }): Promise<{
+        path: string;
+        value: PathValue | undefined;
+    }[]> {
+        return config.paths.map(path => {
+            let value = authorityStorage.getValueAtTime(path);
+            // Get the value, to mutate value, so it doesn't lazily store it,
+            //  so the other size can get the actual value!
+            pathValueSerializer.getPathValue(value);
+            return { path, value };
+        });
+    }
+
+    public async getRandomPaths(config: {
+        count: number;
+        parentWatches: string[];
+    }): Promise<string[]> {
+        let paths: string[][] = [];
+        let pathCount = 0;
+        for (let parentWatch of config.parentWatches) {
+            let childPaths = Array.from(authorityStorage.getPathsFromParent(parentWatch) || []);
+            let sampled = sampleRandom({ count: config.count - paths.length, paths: childPaths });
+            paths.push(sampled);
+            pathCount += sampled.length;
+            if (pathCount >= config.count) {
+                break;
+            }
+        }
+        return paths.flat();
+    }
+
+    public async auditNow(config: {
+        paths: string[];
+    }) {
+        await auditPaths({ paths: config.paths, forceAudit: true });
+    }
+
+
+    public async diskAuditNow() {
+        await diskAuditNow();
+    }
+
+    public async memoryAuditNow() {
+        await fullMemoryAudit();
+    }
+}
+
+export const ValueAuditController = SocketFunction.register(
+    "ValueAuditController-216d498d-9480-4af2-b277-131584958b15",
+    new ValueAuditControllerBase(),
+    () => ({
+        getPathValue: {},
+        getPathValues: {},
+        getRandomPaths: {},
+        auditNow: {},
+
+        diskAuditNow: {},
+        memoryAuditNow: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);
+