querysub 0.2.0

package/src/2-proxy/garbageCollection.ts

@@ -0,0 +1,753 @@
+import { delay } from "socket-function/src/batching";
+import { cache, lazy } from "socket-function/src/caching";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { magenta, blue, red, green, yellow } from "socket-function/src/formatting/logColors";
+import { nextId, sort, binarySearchBasic, timeInDay } from "socket-function/src/misc";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { canHaveChildren } from "socket-function/src/types";
+import { AuthorityPath, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
+import { ReadLock, epochTime, PathValue, authorityStorage, compareTime, getNextTime } from "../0-path-value-core/pathValueCore";
+import { WatchSpec, clientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { logNodeStats } from "../5-diagnostics/nodeMetadata";
+import { getPathFromStr, getPathStr } from "../path";
+import { PromiseObj } from "../promise";
+import { proxyWatcher, atomicRaw, atomic } from "./PathValueProxyWatcher";
+import { rawSchema } from "./pathDatabaseProxyBase";
+import { getProxyPath } from "./pathValueProxy";
+import { runArchiveMover } from "./archiveMoveHarness";
+import { getStorageDir, getSubFolder } from "../fs";
+import fs from "fs";
+import { sha256 } from "js-sha256";
+
+
+export interface AliveChecker<T = unknown> {
+    /** By accessing an object you determine the object passed to isAlive. We consider any object
+     *      to "exist" if it has child keys (even a single key)
+     */
+    object: (wildcard: string) => T;
+    /** Returns a truthy value if alive, falsey if not.
+     *      - If an proxy is returned, we check if it has the placeholder object value
+     *          (proxies virtually always have this, as it is how Object.keys() works).
+     *          This results in the value becoming falsey when the object is removed
+     *          from the parent list.
+     *  NOTE: The value might only have a single heavily nested value, not being a valid object. If the object
+     *      lookups broken, probably just return false, to clean up the straggling values.
+     */
+    isAlive: (value: T, path: string[]) => boolean | unknown;
+
+    /** Alias for lockBasedDelete & unsafeLiveReads */
+    lock?: boolean;
+
+    // Delay deletions by this amount. Used for values that need to last longer than an event write,
+    //  but not forever, such as deleting cached values after a few days.
+    deleteDelayTime?: number;
+    /** Deletes with a lock. If unsafeLiveReads is used this should be used as well.
+     *      - MUCH slower than non-lock deletes.
+     *      - May crash if millions of values are deleted at once.
+     */
+    lockBasedDelete?: boolean;
+    /** Reads live values, in a way that intermediate values may be skipped.
+     *      NOTE: Any values GCed this way will still take the normal time for the deletion
+     *          to be reflected on servers.
+     *      NOTE: Any values NOT in the GC shard will be synced live anyways
+     *          - So... maybe we should turn on unsafe live reads regardless? It's mostly dangerous
+     *              cross shard, and our GC shards will likely line up with our PathValueServer shards,
+     *              so, the cases we make safer are probably less likely to happen than the cases we make
+     *              worse?
+     *              - Although... this IS safer for undefined cleanup. 
+    */
+    unsafeLiveReads?: boolean;
+}
+let aliveCheckers: AliveChecker<any>[] = [];
+
+/** NOTE: Not only does alive checking only operate on flushed values, it also waits for ARCHIVE_FLUSH_LIMIT,
+ *      (which is a lot higher). AND THEN, the changes have to be picked up by a disk audit to propagate to servers.
+ *      AND, when nodes haven't auditted the disk, they might be in a different state. So... it should only
+ *      be used for values users can't access anymore, and so that won't be noticed when they disappear.
+ */
+export function registerAliveChecker<T>(config: AliveChecker<T>) {
+    if (config.lock) {
+        config.lockBasedDelete = true;
+        config.unsafeLiveReads = true;
+    }
+    aliveCheckers.push(config);
+}
+
+export function getAliveCheckers() {
+    return aliveCheckers;
+}
+
+// NOTE: Reusing the watcher (for a trivial function) was about 4X faster,
+//  so this extra code is definitely worth it.
+let getWatchEvaluator = lazy((): {
+    <T>(
+        fnc: () => T,
+        config?: {
+            getReadLocks?: boolean;
+        }
+    ): {
+        pathsRead: string[];
+        parentPathsRead: string[];
+        result: T;
+        readLocks?: ReadLock[];
+    }
+} => {
+    let nextFnc: () => unknown;
+    let nextGetReadLocks: boolean;
+    let lastResult: {
+        pathsRead: string[];
+        parentPathsRead: string[];
+        result: unknown;
+        readLocks?: ReadLock[];
+    } | {
+        error: string;
+    };
+    let watcher = proxyWatcher.createWatcher({
+        static: true,
+        runImmediately: true,
+        unsafeNoLocks: true,
+        canWrite: false,
+        watchFunction() {
+            try {
+                let result = nextFnc();
+                lastResult = {
+                    pathsRead: Array.from(watcher.pendingWatches.paths),
+                    parentPathsRead: Array.from(watcher.pendingWatches.parentPaths),
+                    result
+                };
+                if (nextGetReadLocks) {
+                    lastResult.readLocks = [];
+                    for (let value of watcher.pendingAccesses) {
+                        for (let v of value[1].values()) {
+                            lastResult.readLocks.push({
+                                path: v.path,
+                                startTime: v.time,
+                                // NOTE: endTime will be replaced later, so this value shouldn't matter
+                                endTime: epochTime,
+                                readIsTranparent: !!v.isTransparent,
+                            });
+                        }
+                    }
+                    for (let value of watcher.pendingEpochAccesses) {
+                        for (let v of value[1].values()) {
+                            lastResult.readLocks.push({
+                                path: v,
+                                startTime: epochTime,
+                                // NOTE: endTime will be replaced later, so this value shouldn't matter
+                                endTime: epochTime,
+                                readIsTranparent: true,
+                            });
+                        }
+                    }
+                }
+            } catch (error: any) {
+                lastResult = { error: error.stack || error.toString() };
+            }
+        },
+    });
+    return (fnc, config) => {
+        nextGetReadLocks = config?.getReadLocks || false;
+        nextFnc = fnc;
+        watcher.explicitlyTrigger();
+        if (!lastResult) throw new Error(`Internal error, no result?`);
+        if ("error" in lastResult) throw new Error(lastResult.error);
+        return lastResult as { pathsRead: string[]; parentPathsRead: string[]; result: any };
+    };
+});
+
+const wildcard = Symbol.for("wildcard");
+type PathSelector = {
+    path: (string | typeof wildcard)[];
+};
+
+function sandboxGetPathSelector(config: {
+    checker: AliveChecker;
+}): PathSelector {
+    const checker = config.checker;
+    const wildcardId = nextId();
+    let path = getPathFromStr(getProxyPath(() => checker.object(wildcardId)));
+    return {
+        path: path.map(x => x === wildcardId ? wildcard : x)
+    };
+}
+
+
+const getAllPathMatches = measureWrap(function getAllPathMatches(config: {
+    selectors: PathSelector[]
+    values: PathValue[];
+}): Map<PathSelector, string[][]> {
+    let { selectors, values } = config;
+    let matches = new Map<PathSelector, string[][]>();
+    // NOTE: If this is slow, make a tree of prefixes, and then walk through it using
+    //      the selector path.
+    for (let selector of selectors) {
+        let paths = new Map<string, string[]>();
+        for (let value of values) {
+            let valuePath = getPathFromStr(value.path);
+            //if (selector.path.length !== valuePath.length) continue;
+            // NOTE: We HAVE to match when valuePath.length > selector.path.length. This happens
+            //  if an object is deleted, which is the main reason we want to use this code!
+            //  Instead of just checking the exact path, we consider it to have matched if
+            //  any child path has matched.
+            if (selector.path.length > valuePath.length) continue;
+            let matched = true;
+            for (let i = 0; i < selector.path.length; i++) {
+                if (selector.path[i] !== wildcard && selector.path[i] !== valuePath[i]) {
+                    matched = false;
+                    break;
+                }
+            }
+            if (matched) {
+                let curPath = valuePath.slice(0, selector.path.length);
+                let pathHash = getPathStr(curPath);
+                paths.set(pathHash, curPath);
+            }
+        }
+        if (paths) matches.set(selector, Array.from(paths.values()));
+    }
+    return matches;
+});
+const createRecursiveLookup = measureWrap(function createRecursiveLookup(config: {
+    values: PathValue[];
+}): {
+    getValues: (path: string, parts: string[]) => PathValue[];
+} {
+    let { values } = config;
+    let valuesSorted = values.slice();
+    sort(valuesSorted, x => x.path);
+    return {
+        getValues(path, parts) {
+            let index = binarySearchBasic(valuesSorted, x => x.path, path);
+            if (index < 0) index = ~index;
+            let results: PathValue[] = [];
+            while (index < valuesSorted.length && valuesSorted[index].path.startsWith(path)) {
+                results.push(valuesSorted[index]);
+                index++;
+            }
+            return results;
+        },
+    };
+});
+
+function sandboxedIsAlive(config: {
+    pathStr: string;
+    path: string[];
+    checker: AliveChecker;
+    authority: AuthorityPath;
+    unsyncedPaths: Map<AliveChecker, Map<string, {
+        paths: Set<string>;
+        parentPaths: Set<string>;
+    }>>;
+    errors: { count: number };
+}): boolean {
+    const { pathStr, path, checker, authority, unsyncedPaths } = config;
+
+    let evalResult = getWatchEvaluator()(() => {
+        let proxy = rawSchema()() as any;
+        for (let part of path) {
+            if (!canHaveChildren(proxy)) {
+                proxy = undefined;
+                break;
+            }
+            proxy = proxy[part];
+        }
+        try {
+            return !!atomicRaw(checker.isAlive(proxy, path));
+        } catch {
+            config.errors.count++;
+            return true;
+        }
+    });
+
+    let obj: {
+        paths: Set<string>;
+        parentPaths: Set<string>;
+    } | undefined;
+    function getObj() {
+        if (obj) return obj;
+        let checkerObj = unsyncedPaths.get(checker);
+        if (!checkerObj) {
+            checkerObj = new Map();
+            unsyncedPaths.set(checker, checkerObj);
+        }
+        obj = checkerObj.get(pathStr);
+        if (obj) {
+            return obj;
+        }
+        obj = {
+            paths: new Set(),
+            parentPaths: new Set(),
+        };
+        checkerObj.set(pathStr, obj);
+        return obj;
+    }
+
+    for (let path of evalResult.pathsRead) {
+        if (authorityStorage.isSynced(path)) continue;
+        if (!pathValueAuthority2.isInAuthority(authority, path)) {
+            getObj().paths.add(path);
+        }
+    }
+    for (let path of evalResult.parentPathsRead) {
+        if (authorityStorage.isParentSynced(path)) continue;
+        if (!pathValueAuthority2.isInAuthority(authority, path)) {
+            getObj().parentPaths.add(path);
+        }
+    }
+
+    return evalResult.result;
+}
+function sandboxedGetLocks(config: {
+    pathStr: string;
+    path: string[];
+    checker: AliveChecker;
+}): ReadLock[] {
+    const { pathStr, path, checker } = config;
+
+    let evalResult = getWatchEvaluator()(() => {
+        let value = proxyWatcher.getCallbackPathValue(pathStr);
+        return !!atomic(checker.isAlive(value, path));
+    }, { getReadLocks: true });
+
+    return evalResult.readLocks!;
+}
+
+type ValueFree = {
+    value: PathValue;
+    locks: ReadLock[] | undefined;
+};
+
+// Write wrapper that manages delay times, and calls all functions
+const runAliveCheckerIterationBase = measureWrap(function runAliveCheckerIterationBase(config: {
+    checkPaths: Map<PathSelector, string[][]>;
+    selectors: Map<PathSelector, AliveChecker>;
+    useLocks: boolean;
+    authority: AuthorityPath;
+    getValues: (path: string, parts: string[]) => PathValue[];
+    unsyncedPaths: Map<AliveChecker, Map<string, {
+        paths: Set<string>;
+        parentPaths: Set<string>;
+    }>>;
+    delayedGCRequests: Map<string, {
+        path: string[];
+        freeTime: number;
+    }>,
+}): ValueFree[][] {
+    const now = Date.now();
+    let { checkPaths, selectors, authority, getValues, unsyncedPaths, delayedGCRequests } = config;
+
+    let flatFrees: ValueFree[] = [];
+    let allFrees: ValueFree[][] = [flatFrees];
+
+    let checks = 0;
+    let frees = 0;
+    let allDelayedFrees = 0;
+    let allPreviousDelayedFrees = 0;
+    let errored = { count: 0 };
+
+    let usedGCPaths = new Set<string>();
+
+    console.log(`Checking ${formatNumber(checkPaths.size)} selectors`);
+    for (let [selector, paths] of checkPaths) {
+        let selectorFrees = 0;
+        let delayedFrees = 0;
+        let previousDelayedFrees = 0;
+        console.log(`    ${selector.path.map(x => x === wildcard ? magenta("*") : blue(String(x))).join(".")}`);
+        for (let path of paths) {
+            checks++;
+            let pathStr = getPathStr(path);
+            let checker = selectors.get(selector)!;
+            let isAlive = sandboxedIsAlive({ path, pathStr, checker, authority, unsyncedPaths, errors: errored });
+            if (isAlive) {
+                if (checker.deleteDelayTime) {
+                    delayedGCRequests.delete(pathStr);
+                }
+                continue;
+            }
+
+            if (checker.deleteDelayTime) {
+                // NOTE: Checkers must have a constant PathSelector, so we don't have to worry about
+                //      delayed requests that are matched once, but not the second time. We also
+                //      only check it when checking the selector, not on a timer, so that no matter
+                //      the delay (unless it is very short), we check it against real data twice. This decreases
+                //      false positives? Or something...
+                let request = delayedGCRequests.get(pathStr);
+                let freeTime = now + checker.deleteDelayTime;
+                let existedBefore = !!request;
+                if (!request) {
+                    request = { path, freeTime };
+                    delayedGCRequests.set(pathStr, request);
+                    usedGCPaths.add(pathStr);
+                }
+                if (freeTime < request.freeTime) {
+                    request.freeTime = freeTime;
+                }
+
+                if (request.freeTime > now) {
+                    delayedFrees++;
+                    allDelayedFrees++;
+                    if (existedBefore) {
+                        previousDelayedFrees++;
+                        allPreviousDelayedFrees++;
+                    }
+                    continue;
+                }
+                delayedGCRequests.delete(pathStr);
+            }
+
+            frees++;
+            selectorFrees++;
+
+            // Get recursive paths
+            let recursiveValues = getValues(pathStr, path);
+            if (recursiveValues.length === 0) {
+                console.warn(`No values found for path, which matched selector. How did it match a selector, and not exist? Path: ${pathStr}`);
+                continue;
+            }
+            if (!checker.lockBasedDelete) {
+                for (let value of recursiveValues) {
+                    flatFrees.push({ value, locks: undefined });
+                }
+            } else {
+                let locks = sandboxedGetLocks({ path, pathStr, checker });
+                let curFrees: ValueFree[] = [];
+                for (let value of recursiveValues) {
+                    curFrees.push({ value, locks });
+                }
+                allFrees.push(curFrees);
+            }
+        }
+        console.log(`        Dead objects ${formatNumber(selectorFrees)} / ${formatNumber(paths.length)}`);
+        if (delayedFrees) {
+            console.log(`        Future dead objects: ${formatNumber(delayedFrees)} (${formatNumber(previousDelayedFrees)} previously delayed)`);
+        }
+    }
+
+    console.log(`Dead objects: ${formatNumber(frees)} / ${formatNumber(checks)}`);
+    console.log(`Future dead objects: ${formatNumber(allDelayedFrees)} / ${formatNumber(checks)} (${formatNumber(allPreviousDelayedFrees)} previously delayed)`);
+    if (errored.count) {
+        console.warn(red(`Errored out checks: ${formatNumber(errored.count)}`));
+    }
+
+    return allFrees.filter(x => x.length > 0);
+});
+
+let delayedGCRequestsLists = new Map<string, Map<string, {
+    path: string[];
+    freeTime: number;
+}>>();
+let loadDelayedGCRequests = cache(async (hash: string) => {
+    let cacheGCPath = getStorageDir() + `${hash}_gcRequests.txt`;
+    if (!fs.existsSync(cacheGCPath)) return;
+    try {
+        let contents = await fs.promises.readFile(cacheGCPath, "utf8");
+        let results = JSON.parse(contents) as [string, { path: string[], freeTime: number }][];
+        delayedGCRequestsLists.set(hash, new Map(results));
+        console.log(`Loaded ${results.length} GC requests for ${hash}`);
+    } catch (e) {
+        console.warn(`Failed to load GC requests for ${hash}`, e);
+    }
+});
+async function saveGCRequests(hash: string) {
+    let cacheGCPath = getStorageDir() + `${hash}_gcRequests.txt`;
+    let serialized = JSON.stringify(Array.from(delayedGCRequestsLists.get(hash) || new Map()));
+    await fs.promises.writeFile(cacheGCPath, serialized);
+}
+
+// ALWAYS run a non-lock gc, to remove undefined values, even if no AliveCheckers use it.
+//      Lock frees just create undefines, which will eventually trickly down to a
+//      a gc of the undefined value.
+export async function runAliveCheckerIteration(config?: {
+    // Force, ignoring how much we reduce, merging EVEN if we don't have any frees
+    force?: boolean;
+}) {
+    let outOfShardValues = new Map<string, PathValue>();
+    let pendingWatches: undefined | {
+        paths: Set<string>;
+        parentPaths: Set<string>;
+        promiseObj: PromiseObj<void>;
+    };
+    let watchSpec: WatchSpec = {
+        debugName: "gcOutOfShardWatches",
+        callback: (data) => {
+            for (let value of data.pathSources || []) {
+                outOfShardValues.set(value.path, value);
+            }
+
+            if (pendingWatches) {
+                for (let path of Array.from(pendingWatches.paths)) {
+                    let value = authorityStorage.getValueOrEpochIfSynced(path);
+                    if (value) {
+                        pendingWatches.paths.delete(path);
+                        outOfShardValues.set(path, value);
+                    }
+                }
+                for (let path of Array.from(pendingWatches.parentPaths)) {
+                    if (authorityStorage.isParentSynced(path)) {
+                        pendingWatches.parentPaths.delete(path);
+                        let childPaths = authorityStorage.getPathsFromParent(path);
+                        for (let path of childPaths || []) {
+                            let value = authorityStorage.getValueOrEpochIfSynced(path);
+                            if (value) {
+                                outOfShardValues.set(path, value);
+                            }
+                        }
+                    }
+                }
+
+                if (pendingWatches.paths.size === 0 && pendingWatches.parentPaths.size === 0) {
+                    console.log(`All paths synced, rerunning`);
+                    pendingWatches.promiseObj.resolve();
+                    pendingWatches = undefined;
+                } else {
+                    console.log(`Waiting on paths: ${formatNumber(pendingWatches.paths.size)}, parent parents: ${formatNumber(pendingWatches.parentPaths.size)}`);
+                }
+            }
+        },
+        paths: new Set(),
+        parentPaths: new Set(),
+    };
+
+    try {
+        let checkers = getAliveCheckers();
+
+        function checkerHash(checker: AliveChecker) {
+            return JSON.stringify({ locks: !!checker.lockBasedDelete, live: !!checker.unsafeLiveReads });
+        }
+        let checkerGroups = new Map<string, AliveChecker[]>();
+        for (let checker of checkers) {
+            let hash = checkerHash(checker);
+            let group = checkerGroups.get(hash);
+            if (!group) {
+                checkerGroups.set(hash, group = []);
+            }
+            group.push(checker);
+        }
+        for (let [hash, checkers] of checkerGroups) {
+            hash = sha256(hash);
+            let example = checkers[0];
+            // gclive
+            let type = `gc${example.unsafeLiveReads ? "live" : ""}${example.lockBasedDelete ? "INVALIDUPDATE" : ""}`;
+
+            console.log(`Starting group ${hash} (${type})`);
+
+            await loadDelayedGCRequests(hash);
+            let gcRequestsBase = delayedGCRequestsLists.get(hash);
+            if (!gcRequestsBase) {
+                gcRequestsBase = new Map();
+                delayedGCRequestsLists.set(hash, gcRequestsBase);
+            }
+            let gcRequests = gcRequestsBase;
+
+            let lockWrites: PathValue[] = [];
+            await runArchiveMover({
+                outputType: type,
+                readLiveData: example.unsafeLiveReads,
+                force: config?.force,
+                async runMover(config) {
+                    // Filtering out duplicates and undefineds make the rest of the code simpler. It's also
+                    //  very fast, so we might as well do it, even if we are using locks (which mean our filtered
+                    //  undefined won't result in any deletions).
+                    let values = [
+                        ...config.values,
+                        ...Array.from(outOfShardValues.values()),
+                    ];
+                    let latestValues = new Map<string, PathValue>();
+                    for (let value of values) {
+                        let prev = latestValues.get(value.path);
+                        if (prev && compareTime(value.time, prev.time) < 0) continue;
+                        latestValues.set(value.path, value);
+                    }
+                    values = Array.from(latestValues.values());
+                    values = values.filter(x => !x.canGCValue);
+
+                    let recursiveLookup = createRecursiveLookup({ values });
+
+                    let selectors = new Map<PathSelector, AliveChecker>();
+                    let checkerToSelector = new Map<AliveChecker, PathSelector>();
+
+                    for (let checker of checkers) {
+                        let selector = sandboxGetPathSelector({ checker });
+                        selectors.set(selector, checker);
+                        checkerToSelector.set(checker, selector);
+                    }
+
+                    let remainingPaths = getAllPathMatches({ values, selectors: Array.from(selectors.keys()) });
+
+                    let freeLists: ValueFree[][][] = [];
+                    let loops = 0;
+
+                    while (true) {
+                        loops++;
+                        let unsyncedPaths = new Map<AliveChecker, Map<string, {
+                            paths: Set<string>;
+                            parentPaths: Set<string>;
+                        }>>();
+
+                        let frees = authorityStorage.temporaryOverride(values, () =>
+                            runAliveCheckerIterationBase({
+                                checkPaths: remainingPaths,
+                                selectors,
+                                useLocks: !!example.lockBasedDelete,
+                                authority: config.authority,
+                                getValues: recursiveLookup.getValues,
+                                unsyncedPaths,
+                                delayedGCRequests: gcRequests,
+                            })
+                        );
+
+                        freeLists.push(frees);
+                        if (unsyncedPaths.size === 0) break;
+
+                        remainingPaths = new Map();
+                        for (let [checker, paths] of unsyncedPaths) {
+                            let selector = checkerToSelector.get(checker)!;
+                            remainingPaths.set(selector, Array.from(paths.keys()).map(x => getPathFromStr(x)));
+                        }
+
+                        let checkCount = 0;
+                        let paths = new Set<string>();
+                        let parentPaths = new Set<string>();
+                        for (let map of unsyncedPaths.values()) {
+                            for (let obj of map.values()) {
+                                checkCount++;
+                                for (let path of obj.paths) {
+                                    paths.add(path);
+                                }
+                                for (let path of obj.parentPaths) {
+                                    parentPaths.add(path);
+                                }
+                            }
+                        }
+
+                        for (let path of paths) {
+                            watchSpec.paths.add(path);
+                        }
+                        for (let path of parentPaths) {
+                            watchSpec.parentPaths.add(path);
+                        }
+
+                        console.log(`Out of shard unsynced accesses (loop ${loops})`);
+                        console.log(`    currently unsynced checkers ${formatNumber(unsyncedPaths.size)}`);
+                        console.log(`    currently unsynced objects ${formatNumber(checkCount)}`);
+                        console.log(`    currently unsynced paths ${formatNumber(paths.size)}, unsynced parent paths ${formatNumber(parentPaths.size)}`);
+                        console.log(`    Total path watches ${formatNumber(watchSpec.paths.size)}, total parent path watches ${formatNumber(watchSpec.parentPaths.size)}`);
+                        console.log(`    Total out of shard values ${formatNumber(outOfShardValues.size)}`);
+                        logNodeStats("archives|Unsynced Checkers", formatNumber)(paths.size);
+                        logNodeStats("archives|Unsynced Paths", formatNumber)(unsyncedPaths.size);
+                        logNodeStats("archives|Unsynced Parent Paths", formatNumber)(parentPaths.size);
+
+                        clientWatcher.setWatches(watchSpec);
+
+                        let wait = new PromiseObj();
+                        pendingWatches = {
+                            paths,
+                            parentPaths,
+                            promiseObj: wait,
+                        };
+                        await Promise.race([delay(15000), wait.promise]);
+                    }
+                    {
+                        let totalFreeCount = 0;
+                        for (let frees of freeLists) {
+                            for (let freeList of frees) {
+                                totalFreeCount += freeList.length;
+                            }
+                        }
+                        console.log(green(`Free now: ${formatNumber(totalFreeCount)} / ${formatNumber(values.length)}`));
+                        logNodeStats("archives|Free now", formatNumber)(totalFreeCount);
+                    }
+                    {
+                        let pendingFrees = gcRequests.size;
+                        console.log(yellow(`Stored future frees: ${formatNumber(pendingFrees)} / ${formatNumber(values.length)}`));
+
+                        let futureByDayCount = new Map<number, number>();
+                        let now = Date.now();
+                        for (let obj of gcRequests.values()) {
+                            let day = Math.floor((obj.freeTime - now) / timeInDay);
+                            let value = futureByDayCount.get(day) || 0;
+                            value++;
+                            futureByDayCount.set(day, value);
+                        }
+                        let dayCounts = Array.from(futureByDayCount);
+                        sort(dayCounts, x => x[0]);
+                        for (let [day, count] of dayCounts) {
+                            console.log(`    < ${day} days: ${formatNumber(count)}`);
+                        }
+
+                        logNodeStats("archives|Stored future frees", formatNumber)(pendingFrees);
+                    }
+
+                    if (example.lockBasedDelete) {
+                        let lockCount = 0;
+                        for (let frees of freeLists) {
+                            for (let freeList of frees) {
+                                // We need to extend our value write time as later as possible, otherwise
+                                //      it will be rejected (it needs to be set almost immediately before we commit it).
+                                //  This is still a BIT dangerous, because if we have 1 million values we might
+                                //      not iterate in under 30 seconds, but... it's good enough for now.
+                                //      (Also, at least some will be accepted, so we will make progress).
+                                let time = getNextTime();
+                                let locks = freeList[0].locks!;
+                                for (let lock of locks) {
+                                    lock.endTime = time;
+                                    lockCount++;
+                                }
+                                for (let free of freeList) {
+                                    lockWrites.push({
+                                        path: free.value.path,
+                                        time,
+
+                                        locks,
+                                        value: undefined,
+                                        isValueLazy: false,
+
+                                        canGCValue: true,
+                                        event: undefined,
+                                        isTransparent: true,
+                                        valid: true,
+                                        source: undefined,
+                                        lockCount: locks.length,
+                                        updateCount: 0,
+                                    });
+                                }
+                            }
+                        }
+                        console.log(`Committing with ${formatNumber(lockCount)} locks`);
+                        return { newValues: "abort" };
+                    }
+
+                    let allFrees = new Set(freeLists.flat().flat().map(x => x.value));
+                    values = values.filter(x => !allFrees.has(x));
+
+                    // NOTE: We don't notify any servers about this. We will eventually have to write
+                    //  audit code to periodically check against the disk to get updates.
+                    return {
+                        newValues: {
+                            "": values,
+                        },
+                    };
+                },
+            });
+
+            await saveGCRequests(hash);
+
+            if (lockWrites.length > 0) {
+                // We are just going to hope committing 100K raw writes doesn't break anything. If it does...
+                //      I'm sorry. But then again, "lock"-type gc is probably overkill anyways. How much user-accessible
+                //      data do you have that you need to free it? GCing is general for unreachable data, created
+                //      temporarily to communicate between servers, such as temporary images, or... to delete
+                //      lookup deleted keys, which shouldn't need locks anyways.
+                pathValueCommitter.commitValues(lockWrites, undefined);
+                await pathValueCommitter.waitForValuesToCommit();
+                console.log(`Committed ${formatNumber(lockWrites.length)} writes`);
+            }
+        }
+    } finally {
+        watchSpec.paths.clear();
+        watchSpec.parentPaths.clear();
+        outOfShardValues.clear();
+        clientWatcher.setWatches(watchSpec);
+    }
+}