querysub 0.2.0

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -0,0 +1,978 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { errorToUndefined, errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
+import { PromiseObj } from "../promise";
+import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, isNodeDiscoveryLogging, isOwnNodeId, onNodeDiscoveryReady, watchDeltaNodeIds, watchNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { PathValueController } from "./PathValueController";
+import { MAX_ACCEPTED_AUTHORITY_STARTUP_TIME, PathValueSnapshot, STARTUP_CUTOFF_TIME, authorityStorage, matchesParentRangeFilterPart } from "./pathValueCore";
+import { pathValueArchives } from "./pathValueArchives";
+import { deepCloneJSON, isNode, sha256Hash, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { delay, runInfinitePoll } from "socket-function/src/batching";
+import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { getNodeIdFromLocation, getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { appendToPathStr, getPathDepth, getPathFromStr, getPathIndex, getPathStr, getPathStr1, getPathStr2 } from "../path";
+import { MaybePromise } from "socket-function/src/types";
+import { isClient, isServer } from "../config2";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { sha256 } from "js-sha256";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { cache, cacheLimited } from "socket-function/src/caching";
+import { IdentityController_getCurrentReconnectNodeIdAssert, IdentityController_getReconnectNodeIdAssert } from "../-c-identity/IdentityController";
+import { getBufferFraction, getBufferInt, getShortNumber } from "../bits";
+import { devDebugbreak, isDevDebugbreak } from "../config";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+
+export const LOCAL_DOMAIN = "LOCAL";
+export const LOCAL_DOMAIN_PATH = getPathStr1(LOCAL_DOMAIN);
+
+const POLL_RATE = 5000;
+const MAX_RECONNECT_TIME = timeInMinute * 15;
+const RECONNECT_POLL_INTERVAL = 10000;
+
+// Poll nodes that appear dead. Without this, if the internet goes down, we might forever ignore nodes.
+const RECOVERY_POLL_INTERVAL = timeInMinute;
+
+// NOTE: We don't implicitly include parents, as when we add multiple specific paths it is easier
+//  to do it off of a tree of AuthorityPathPart2s, instead of trying to store a whole tree's worth
+//  of into in a single AuthorityPathPart2.
+export type AuthorityPath = {
+    // NOTE: Object.keys() accesses "" might sometimes be considered a wildcard.
+
+    // MATCHES path.startsWith(pathPrefix)
+    pathPrefix: string;
+    // AND MATCHES !path.startsWith(excludedChildren[i]) || path === excludedChildren[i]
+    //  (As in, this only excludes descendants, not the exact child matches)
+    excludedChildren?: string[];
+    // See matchesParentRangeFilterPart
+    hash?: {
+        // Depth MUST equal getPathDepth(pathPrefix), and is only stored for faster matching
+        depth: number;
+        start: number;
+        end: number;
+    };
+    /* Causes any path parts that are "" to be considered wildcards, and allow matching any
+        path (both in pathPrefix and excludedChildren).
+        NOTE: The only reason to NOT set this is because wild card checks are slow.
+    */
+    emptyIsWildcard?: boolean;
+};
+
+export type AuthorityObj = {
+    nodeId: string;
+    authorityPaths: AuthorityPath[];
+    isReadReady: number;
+    createTime: number;
+    self?: boolean;
+};
+
+type WatchedAuthorityObj = AuthorityObj & {
+    // Waits until ready OR throws, if MAX_ACCEPTED_AUTHORITY_STARTUP_TIME is exceeded
+    onReady: PromiseObj<void>;
+};
+class NodePathAuthorities {
+    // nodeId =>
+    private authorities = new Map<string, WatchedAuthorityObj>();
+    private disconnectedAuthorities = new Map<string, WatchedAuthorityObj>();
+    private disconnectCounts = new WeakMap<WatchedAuthorityObj, number>();
+
+    private selfAuthorities: AuthorityPath[] = [];
+
+    public debug_getAuthority(nodeId: string) {
+        return this.authorities.get(nodeId);
+    }
+
+    public getSelfAuthorities(): AuthorityPath[] {
+        return this.selfAuthorities;
+    }
+
+    public getAuthorityPaths(nodeId: string) {
+        return this.authorities.get(nodeId)?.authorityPaths;
+    }
+
+    public readonly createTime = Date.now();
+    constructor() {
+        // NOTE: WHILE calling this function at the module level (because we this is a singleton) is bad,
+        //  it SHOULD be fine. "getOwnNodeId" is part of level "-f-node-discovery", and we are level "0-path-value-core".
+        //  It should be ready before us, and it should never be allowed to depend on us.
+        let originalOwnId = getOwnNodeId();
+        let selfOnReady = new PromiseObj();
+        this.authorities.set(getOwnNodeId(), {
+            nodeId: getOwnNodeId(),
+            authorityPaths: [],
+            isReadReady: 0,
+            createTime: this.createTime,
+            self: true,
+            onReady: selfOnReady,
+        });
+        if (isServer()) {
+            void SocketFunction.mountPromise.finally(() => {
+                // Move over to actual nodeId (which only exists after we mount)
+                let newOwnId = getOwnNodeId();
+                if (newOwnId !== originalOwnId) {
+                    let obj = this.authorities.get(originalOwnId);
+                    if (obj) {
+                        this.authorities.delete(originalOwnId);
+                        this.authorities.set(newOwnId, obj);
+                    }
+                }
+                let promise = this.baseBecomeAuthority(this.createTime);
+                promise.catch(this.routingReady.reject);
+                selfOnReady.resolve(promise);
+                logErrors(promise);
+            });
+        } else {
+            selfOnReady.resolve();
+            let promise = this.watchAuthorityPaths();
+            promise.catch(this.routingReady.reject);
+            logErrors(promise);
+        }
+    }
+
+    private reconnectLoop = runInfinitePoll(RECONNECT_POLL_INTERVAL, async () => {
+        if (!isNode()) return;
+        let disconnectedAuthorities = Array.from(this.disconnectedAuthorities.values());
+        await Promise.allSettled(
+            disconnectedAuthorities.map(async x => {
+                let isAliveAgain = await timeoutToUndefinedSilent(5000, PathController.nodes[x.nodeId].isReadReady());
+                if (isAliveAgain) {
+                    this.disconnectedAuthorities.delete(x.nodeId);
+                    // NOTE: We don't wait for reconnections if we fail to find an authority or anything, because...
+                    //      it is VERY unlikely for nodes to reconnect. HOWEVER, RemoteWatcher polls forever
+                    //      for orphaned watches, so reconnecting here will result in the node being reused
+                    //      somewhat quickly.
+                    this.authorities.set(x.nodeId, x);
+                    this.disconnectCounts.delete(x);
+                    console.log(green(`Reconnected to ${x.nodeId}`));
+                } else {
+                    let prevCount = this.disconnectCounts.get(x) || 0;
+                    prevCount++;
+                    // If it takes too long, give up, and stop polling.
+                    if (prevCount > (MAX_RECONNECT_TIME / RECONNECT_POLL_INTERVAL)) {
+                        this.disconnectedAuthorities.delete(x.nodeId);
+                        this.disconnectCounts.delete(x);
+                        return;
+                    }
+                    this.disconnectCounts.set(x, prevCount);
+
+                }
+            })
+        );
+    });
+
+    // NOTE: Must be called BEFORE mounting
+    public becomePathAuthority(authority: AuthorityPath) {
+        if (SocketFunction.isMounted()) throw new Error(`Cannot call becomePathAuthority after mounting`);
+        authority = deepCloneJSON(authority);
+        if (authority.hash) {
+            authority.hash.depth = getPathDepth(authority.pathPrefix);
+        }
+        if (!this.selfAuthorities.includes(authority)) {
+            this.selfAuthorities.push(authority);
+            // Longest first is more efficient
+            sort(this.selfAuthorities, x => -x.pathPrefix.length);
+        }
+        let authorityObj = this.authorities.get(getOwnNodeId());
+        if (!authorityObj) {
+            throw new Error(`Authority object not found for self. Should be impossible, but maybe our own id changed. Own id is ${getOwnNodeId()}, we have authorities for: ${Object.keys(this.authorities).join(", ")}`);
+        }
+        let hash = JSON.stringify(authority);
+        if (!authorityObj.authorityPaths.some(x => JSON.stringify(x) === hash)) {
+            authorityObj.authorityPaths.push(authority);
+            sort(authorityObj.authorityPaths, x => -x.pathPrefix.length);
+        }
+
+        console.log(blue(`Becoming an authority for ${getArchiveDirectory(authority)}`));
+    }
+
+    private isReadReady = 0;
+    public isSelfReadReady(): number {
+        return this.isReadReady;
+    }
+
+    @measureFnc
+    private async watchAuthorityPaths() {
+        await onNodeDiscoveryReady();
+
+        onReadyReady = (nodeId, time) => {
+            let obj = this.authorities.get(nodeId);
+            if (!obj) {
+                // HACK: Sometimes when a node is first added we can't identify it yet. I THINK this is because
+                //  we haven't learned to trust the key, or... something? Hmm...
+                // ingestNewNodeIds([nodeId], []);
+                return;
+            }
+            obj.isReadReady = time;
+            obj.onReady.resolve();
+        };
+
+        let first = true;
+        let firstPromise = new PromiseObj<unknown>();
+
+        const ingestNewNodeIds = (newNodeIds: string[], removedNodeIds: string[]) => {
+            for (let nodeId of removedNodeIds) {
+                this.authorities.delete(nodeId);
+            }
+
+            let nodeIds = newNodeIds;
+            nodeIds = nodeIds.filter(nodeId => !isOwnNodeId(nodeId));
+            let newNodes = nodeIds.filter(nodeId => !this.authorities.has(nodeId));
+            let timeout = first ? POLL_RATE : 0;
+            first = false;
+
+            let promise = Promise.allSettled(newNodes.map(async nodeId => {
+                if (this.authorities.has(nodeId)) return;
+
+                let loc = getNodeIdLocation(nodeId);
+                let ourLoc = getNodeIdLocation(getOwnNodeId());
+                if (loc && loc.address.startsWith("127-0-0-1") && loc.port === ourLoc?.port) {
+                    // Ignore it if it's just us, using the special localhost address
+                    return;
+                }
+
+                let createTime = await errorToUndefinedSilent(PathController.nodes[nodeId].getCreateTime());
+                if (createTime === undefined) {
+                    // Don't log for 127-0-0-1, as it usually fails, and is mostly a development optimization
+                    if (!nodeId.includes("127-0-0-1")) {
+                        diskLog(`Node didn't respond to getCreateTime`, { nodeId });
+                    }
+                    return;
+                }
+                console.log(blue(`Identifying ${nodeId} as a path authority`));
+                this.authorities.set(nodeId, {
+                    nodeId,
+                    self: nodeId.startsWith("127-0-0-1."),
+                    authorityPaths: [],
+                    isReadReady: 0,
+                    createTime,
+                    onReady: new PromiseObj(),
+                });
+                let obj = this.authorities.get(nodeId)!;
+                SocketFunction.onNextDisconnect(nodeId, () => {
+                    this.authorities.delete(nodeId);
+                    this.disconnectedAuthorities.set(nodeId, obj);
+                });
+                let finished = false;
+                try {
+                    if (timeout) {
+                        let timeoutObj = new PromiseObj();
+                        setTimeout(() => {
+                            timeoutObj.resolve();
+                            if (!finished && timeout) {
+                                console.warn(yellow(`Timeout after ${formatTime(timeout)} while identifying ${nodeId}`));
+                            }
+                        }, timeout);
+                        await Promise.race([
+                            timeoutObj.promise,
+                            errorToUndefined(this.forceCheckForReadReady(nodeId)),
+                        ]);
+                    }
+                    // In case we miss the broadcast, poll to see if it becomes ready
+                    if (!obj.isReadReady) {
+                        ignoreErrors((async () => {
+                            while (!obj.isReadReady && this.authorities.has(nodeId)) {
+                                await delay(timeInSecond * 10);
+                                if (obj.isReadReady) break;
+                                await errorToUndefinedSilent(this.forceCheckForReadReady(nodeId));
+                            }
+                        })());
+                    }
+                } finally {
+                    finished = true;
+                }
+            }));
+            logErrors(firstPromise);
+            firstPromise.resolve(promise);
+        };
+
+        let time = Date.now();
+        watchDeltaNodeIds(obj => ingestNewNodeIds(obj.newNodeIds, obj.removedNodeIds));
+        await firstPromise.promise;
+
+        runInfinitePoll(RECOVERY_POLL_INTERVAL, async () => {
+            let nodes = await getAllNodeIds();
+            let nonExistentNodes: string[] = [];
+            for (let node of nodes) {
+                if (this.authorities.has(node)) continue;
+                if (this.disconnectedAuthorities.has(node)) continue;
+                nonExistentNodes.push(node);
+            }
+            // Pretend all dead nodes are new
+            ingestNewNodeIds(nonExistentNodes, []);
+        });
+
+        let readyCount = 0;
+        for (let authority of this.authorities.values()) {
+            if (!authority.isReadReady) continue;
+            if (authority.authorityPaths.length === 0) continue;
+            readyCount++;
+            console.log(`    ${green(`(ready)`)} ${authority.nodeId}`);
+        }
+        console.log(green(`Finished loading all ${this.authorities.size} path authorities in ${formatTime(Date.now() - time)}. Ready authorities: ${readyCount}`));
+
+
+        this.routingReady.resolve();
+        this.waitUntilRoutingIsReadyBase = () => undefined;
+    }
+
+    private async forceCheckForReadReady(nodeId: string) {
+        let obj = this.authorities.get(nodeId);
+        if (!obj) return;
+        if (obj.isReadReady) return;
+        if (isNodeDiscoveryLogging()) {
+            console.log(blue(`Starting to identify ${nodeId} as a path authority`));
+        }
+        // now BEFORE we poll, so skew towards lower times, as higher times results in us discarding the node
+        //  as being out of date.
+        let now = Date.now();
+        let isReadReady = await errorToUndefinedSilent(PathController.nodes[nodeId].isReadReady());
+        // If the node is gone, delete it from authorities
+        if (isReadReady === undefined) {
+            console.error(yellow(`Node errored out, removing from authorities ${nodeId}`));
+            this.authorities.delete(nodeId);
+            return;
+        }
+        if (!isReadReady && obj.createTime + MAX_ACCEPTED_AUTHORITY_STARTUP_TIME < now) {
+            let errorMessage = `Node took too long to become ready. Removing from authorities. ${nodeId}`;
+            obj.onReady.reject(new Error(errorMessage));
+            console.error(red(errorMessage));
+            this.authorities.delete(nodeId);
+            return;
+        }
+
+        obj.isReadReady = isReadReady;
+        obj.onReady.resolve();
+
+        obj.authorityPaths = await PathController.nodes[nodeId].getAuthorityPaths();
+        if (obj.authorityPaths.length > 0) {
+            console.log(blue(`Identified ${isReadReady ? "(ready)" : yellow("(loading)")} ${nodeId} as a path authority for:`));
+            for (let authorityPath of obj.authorityPaths) {
+                console.log(`    ${getArchiveDirectory(authorityPath)}`);
+            }
+        }
+    }
+
+    @measureFnc
+    private async baseBecomeAuthority(createTime: number) {
+        await this.watchAuthorityPaths();
+
+        // NOTE: This technically races, as even if all NEW values will be sent to us, there may be on the
+        //  write that we can't read in any way (disk or memory reads), which we just have to wait to flush.
+        //  BUT... this is very uncommon, as virtually all of the time all network writes will finish
+        //      before we can flush to backblaze...
+
+        let authoritySources: {
+            authorityPath: AuthorityPath;
+            authorities: AuthorityObj[];
+        }[] = [];
+
+        // Find sources, waiting for all to be ready. Memory cleanup requires 
+        while (true) {
+            authoritySources = [];
+            for (let authorityPath of this.getSelfAuthorities()) {
+                let sources: AuthorityObj[] = [];
+                for (let otherAuthObj of this.authorities.values()) {
+                    if (otherAuthObj.self) continue;
+                    // Only load from authorites newer than us
+                    if (otherAuthObj.createTime > createTime) continue;
+                    let matchingAuth = Object.values(otherAuthObj.authorityPaths).find(x => authoritiesMightOverlap(x, authorityPath));
+                    if (!matchingAuth) continue;
+                    sources.push(otherAuthObj);
+                }
+                authoritySources.push({ authorityPath, authorities: sources });
+            }
+
+            let waitForAuthorities = authoritySources.flatMap(x => x.authorities).filter(x => !x.isReadReady);
+            if (waitForAuthorities.length === 0) break;
+            console.log(yellow(`Waiting for ${waitForAuthorities.length} authorities to be ready:`));
+            // Log the paths we are waiting for
+            for (let source of authoritySources) {
+                if (source.authorities.every(x => x.isReadReady)) continue;
+                console.log(yellow(`    ${getArchiveDirectory(source.authorityPath)}`));
+            }
+            await delay(POLL_RATE);
+            // NOTE: forceCheckForReadReady will remove dead authorities.
+            await Promise.allSettled(waitForAuthorities.map(x => this.forceCheckForReadReady(x.nodeId)));
+            // If any authorities are not ready (and have been matched, wait, and then check them again).
+        }
+
+        // Load all values from memory
+        let allSnapshots: PathValueSnapshot[] = [];
+        for (let source of authoritySources) {
+            console.log(blue(`Loading authority path ${getArchiveDirectory(source.authorityPath)}`));
+            for (let otherAuthObj of source.authorities) {
+                let nodeId = otherAuthObj.nodeId;
+                console.log(blue(`    Loading snapshot from ${nodeId}`));
+                // NOTE: getSnapshot automatically filters to those asked for on authorityPath
+                let snapshotBuffers = await PathValueController.nodes[nodeId].getSnapshot({ authorityPath: source.authorityPath });
+                let totalSize = snapshotBuffers.reduce((a, b) => a + b.length, 0);
+                console.log(green(`    Loaded ${formatNumber(snapshotBuffers.length)} buffers, total ${formatNumber(totalSize)}B`));
+                let pathValues = await pathValueSerializer.deserialize(snapshotBuffers);
+                console.log(green(`    Loaded ${formatNumber(pathValues.length)} values`));
+                let snapshot: PathValueSnapshot = {
+                    values: {},
+                };
+                for (let pathValue of pathValues) {
+                    let path = pathValue.path;
+                    let values = snapshot.values[path];
+                    if (!values) {
+                        snapshot.values[path] = values = [];
+                    }
+                    values.push(pathValue);
+                }
+                allSnapshots.push(snapshot);
+            }
+
+            let requireArchivesLoad = !source.authorities.some(x => x.authorityPaths.some(y =>
+                // TODO: Deal with subsets, instead of only using an exact comparison
+                JSON.stringify(y) === JSON.stringify(source.authorityPath)
+            ));
+            if (requireArchivesLoad) {
+                if (source.authorities.length > 0) {
+                    console.log(yellow(`Loading from disk for ${getArchiveDirectory(source.authorityPath)}, even though there are existing authorities. This is likely technically unnecessary, but presently done because we haven't implemented AuthorityPath subset code. If this takes an excessive amount of time, we should implement the correct AuthorityPath subset code.`));
+                } else {
+                    console.log(blue(`Loading from disk for ${getArchiveDirectory(source.authorityPath)}`));
+                }
+                let snapshot = await pathValueArchives.loadValues(source.authorityPath);
+                allSnapshots.push(snapshot);
+            }
+        }
+
+        for (let snapshot of allSnapshots) {
+            authorityStorage.ingestSnapshot(snapshot);
+        }
+
+        let readyTime = Date.now();
+        let timeToReady = readyTime - this.createTime;
+        if (timeToReady > MAX_ACCEPTED_AUTHORITY_STARTUP_TIME) {
+            console.error(red(`Took too long to become an authority (${formatTime(timeToReady)}), other nodes will not accept us as an authority, killing process now`));
+            process.exit();
+        }
+        this.isReadReady = readyTime;
+        for (let nodeId of await getAllNodeIds()) {
+            ignoreErrors(PathController.nodes[nodeId].broadcastReadReady(readyTime));
+        }
+
+        if (this.getSelfAuthorities().length > 0) {
+            console.log(green(`Became an authority for:`));
+            for (let authorityPath of this.getSelfAuthorities()) {
+                console.log(green(`    ${getArchiveDirectory(authorityPath)}`));
+            }
+        }
+    }
+
+    private waited = false;
+    private routingReady = new PromiseObj();
+    @measureFnc
+    public waitUntilRoutingIsReady() {
+        this.waited = true;
+        return this.waitUntilRoutingIsReadyBase();
+    }
+    // NOTE: For some reason, having to wait on the promise every time is really slow (~1ms). So... once it is done,
+    //  we make it return undefined, avoiding this issue. It probably isn't REALLY slow, and is probably just a measurement
+    //  error, but... oh well...
+    private waitUntilRoutingIsReadyBase: () => MaybePromise<void> = async () => {
+        return this.routingReady.promise;
+    };
+    public assertRoutingIsReady() {
+        let waitPromise = this.waitUntilRoutingIsReadyBase();
+        if (waitPromise !== undefined) {
+            throw new Error(`waitUntilRoutingIsReady was not called`);
+        }
+    }
+
+    public isSelfAuthority(path: string): boolean {
+        if (this.isLocalPath(path)) return true;
+        return this.selfAuthorities.some(x => isInAuthority(x, path));
+    }
+    public isInAuthority(authority: AuthorityPath, path: string): boolean {
+        return isInAuthority(authority, path);
+    }
+    public isLocalPath(path: string): boolean {
+        return path.startsWith(LOCAL_DOMAIN_PATH);
+    }
+
+
+    /** Not a full path, it is expected to be nested under the main PathValue archive
+     *      (or wherever you want to store files).
+     */
+    public getArchiveDirectory(authorityPath: AuthorityPath): string {
+        return getArchiveDirectory(authorityPath);
+    }
+    public mightMatchArchiveDirectory(authorityPath: AuthorityPath, directory: string): boolean {
+        return mightMatchArchiveDirectory(authorityPath, directory);
+    }
+
+    public getSelfArchiveAuthority(path: string): AuthorityPath | undefined {
+        for (let authorityPath of this.getSelfAuthorities()) {
+            if (isInAuthority(authorityPath, path)) {
+                return authorityPath;
+            }
+        }
+        return undefined;
+    }
+
+    @measureFnc
+    private getReadObjs(path: string, config?: { allowUnready?: boolean }) {
+        return Array.from(this.authorities.values())
+            // If it will never be ready... just ignore it
+            .filter(x => !x.onReady.rejected)
+            .filter(x => config?.allowUnready || x.isReadReady)
+            .filter(x =>
+                Object.values(x.authorityPaths).some(authority =>
+                    isInAuthority(authority, path)
+                )
+            );
+    }
+    public getReadNodes(path: string): string[] {
+        if (isClient()) {
+            return [getBrowserUrlNode()];
+        }
+        if (!this.waited) throw new Error(`waitUntilRoutingIsReady must be called before getReadNodes`);
+        return this.getReadObjs(path).map(x => x.nodeId);
+    }
+
+    private pickAuthorityNode(objs: AuthorityObj[] | undefined): AuthorityObj | undefined {
+        if (!objs) return undefined;
+        // Prefer local nodes if they exist, and we are developing
+        if (isDevDebugbreak()) {
+            let local = objs.find(x => x.nodeId.startsWith("127-0-0-1."));
+            if (local) {
+                return local;
+            }
+        }
+        return objs[Math.floor(Math.random() * objs.length)];
+    }
+
+    private logWaitingForNode = cache((nodeId: string) => {
+        console.log(yellow(`Waiting for ${nodeId} to be ready`));
+    });
+    public async getSingleReadNodePromise(path: string): Promise<string> {
+        await this.waitUntilRoutingIsReady();
+        let nodeId = this.getSingleReadNodeSync(path);
+        // 1) See if there IS a node, but maybe it is just not ready yet
+        if (!nodeId) {
+            let objs = this.getReadObjs(path, { allowUnready: true });
+            nodeId = this.pickAuthorityNode(objs)?.nodeId;
+            if (nodeId) {
+                let obj = this.authorities.get(nodeId)!;
+                this.logWaitingForNode(nodeId);
+
+                // Only wait a few seconds, and then try again. This solves the issue of a node loading,
+                //      and then taking forever, while other nodes have already become ready since we
+                //      started waiting
+                await Promise.race([
+                    // Ignore errors, we will log it somewhere else, and we shouldn't fail the call
+                    //      just because one node errored out
+                    obj.onReady.promise.catch(() => { }),
+                    delay(5000),
+                ]);
+                // Call again, as either the node is ready, or we should wait
+                //      for another node to be ready (or no nodes are even loading!)
+                return this.getSingleReadNodePromise(path);
+            }
+        }
+        if (!nodeId && Date.now() < STARTUP_CUTOFF_TIME) {
+            // Wait a bit, then try again
+            await delay(5000);
+            return this.getSingleReadNodePromise(path);
+        }
+        if (!nodeId) {
+            throw new Error(`No node found for path ${path}`);
+        }
+        return nodeId;
+    }
+    public getSingleReadNodeSync(path: string): string | undefined {
+        if (this.isSelfAuthority(path)) {
+            return getOwnNodeId();
+        }
+        if (isClient()) {
+            return getBrowserUrlNode();
+        }
+        if (!this.waited) throw new Error(`waitUntilRoutingIsReady must be called before getSingleReadNode`);
+        let objs = this.getReadObjs(path);
+        return this.pickAuthorityNode(objs)?.nodeId;
+    }
+
+    public getChildReadNodes(path: string, config?: {
+        preferredNodeIds?: Map<string, unknown>;
+    }): {
+        // NOTE: If at all possible, we will cover all ranges. Node of the returned nodes will be redundant.
+        //  - Sorted by range.start
+        nodes: {
+            nodeId: string;
+            // The range of hashes this node owns, for the child keys of path
+            //  (If the node doesn't restrict the range, it will just be { start: 0, end: 1 })
+            range: { start: number; end: number };
+        }[];
+    } {
+        if (this.isSelfAuthority(path)) {
+            return { nodes: [{ nodeId: getOwnNodeId(), range: { start: 0, end: 1 } }] };
+        }
+        if (isClient()) {
+            return { nodes: [{ nodeId: getBrowserUrlNode(), range: { start: 0, end: 1 } }] };
+        }
+
+        let wildcardPath = appendToPathStr(path, "");
+
+        let pathDepth = getPathDepth(path);
+
+        // Sweep over the nodes, sorted by start, adding each node with an end > the previous end.
+        //  - Whenever we have duplicates with the same start, pick one according to our pick algorithm.
+        //  ALSO, if one of the options is in preferredNodeIds, pick it automatically
+
+        let allNodes: {
+            nodeId: AuthorityObj;
+            range: { start: number; end: number };
+        }[] = [];
+        for (let authorityObj of this.authorities.values()) {
+            if (!authorityObj.isReadReady) continue;
+            for (let path of Object.values(authorityObj.authorityPaths)) {
+                // NOTE: The only reason we don't ALWAYS check wildcards is for speed, so forcing true here is fine...
+                if (!isInAuthority({ ...path, emptyIsWildcard: true }, wildcardPath)) continue;
+                let range = { start: 0, end: 1 };
+                if (path.hash?.depth === pathDepth) {
+                    range = path.hash;
+                }
+                allNodes.push({ nodeId: authorityObj, range });
+            }
+        }
+
+        let pickedNodes: {
+            nodeId: string;
+            range: { start: number; end: number };
+        }[] = [];
+
+        sort(allNodes, x => x.range.start);
+        let allNodesByStart: {
+            start: number;
+            nodes: {
+                nodeId: AuthorityObj;
+                range: { start: number; end: number };
+            }[];
+        }[] = [];
+
+        for (let node of allNodes) {
+            let last = allNodesByStart[allNodesByStart.length - 1];
+            if (!last || last.start !== node.range.start) {
+                last = { start: node.range.start, nodes: [] };
+                allNodesByStart.push(last);
+            }
+            last.nodes.push(node);
+        }
+
+        let lastEnd = -1;
+        for (let { start, nodes } of allNodesByStart) {
+            nodes = nodes.filter(x => x.range.end > lastEnd);
+            if (nodes.length === 0) continue;
+            let preferredNodes = nodes.filter(x => config?.preferredNodeIds?.has(x.nodeId.nodeId));
+            if (preferredNodes.length > 0) {
+                nodes = preferredNodes;
+            }
+            let node = this.pickAuthorityNode(nodes.map(x => x.nodeId));
+            if (node) {
+                let obj = nodes.find(x => x.nodeId.nodeId === node?.nodeId)!;
+                pickedNodes.push({ nodeId: node.nodeId, range: obj.range });
+                lastEnd = obj.range.end;
+            }
+        }
+
+        // At the end do a check for purely redundant nodes. This can happen if we have tiling overlaps. We can determine
+        //  this by comparing to the previous and next nodes.
+        for (let i = 1; i < pickedNodes.length - 1; i++) {
+            let prev = pickedNodes[i - 1];
+            let next = pickedNodes[i + 1];
+            // If our neighbors touch, then we are not needed
+            if (prev.range.end >= next.range.start) {
+                pickedNodes.splice(i, 1);
+                // Try i again, as the next node might itself be redundant, using the same prev
+                i--;
+            }
+        }
+
+        sort(pickedNodes, x => x.range.start);
+        return { nodes: pickedNodes };
+    }
+
+    // NOTE: Nodes become writeable before they become readable, so they have to be guaranteed to have
+    //  all values to be readable, but need to be writeable immediately (otherwise they will never have all values)
+    //  - However, we might eventually create read only nodes, which are just replicas of a write nodes,
+    //      so we can reduce our redundant backblaze / storage throughput (it might also
+    //      be faster, or allow us to run a less powerful node which only reads?)
+    public async getWriteNodes(path: string): Promise<string[]> {
+        if (this.isLocalPath(path)) return [getOwnNodeId()];
+        // IMPORTANT! Even if we are the authority, this doesn't mean other nodes aren't also the authority,
+        //  so we have to let it propagate to all nodes.
+        //if (this.isSelfAuthority(path)) return [getOwnNodeId()];
+        if (isClient()) {
+            return [getBrowserUrlNode()];
+        }
+        await this.waitUntilRoutingIsReady();
+
+        let result = Array.from(this.authorities.values())
+            .filter(x => !x.onReady.rejected)
+            .filter(x => Object.values(x.authorityPaths).some(authorityPath => isInAuthority(authorityPath, path)))
+            .map(x => x.nodeId);
+        if (result.length === 0) {
+            await this.getSingleReadNodePromise(path);
+        }
+        return result;
+    }
+}
+
+
+// Length prefixes to escape parts
+function encodeNiceArray(arr: string[], delimit = "."): string {
+    return arr.map(value => {
+        if (arr.includes(delimit) || !value) {
+            return delimit + value.length + delimit + value;
+        }
+        return value;
+    }).join(delimit);
+}
+function decodeNiceArray(str: string, delimit = "."): string[] {
+    let output: string[] = [];
+    let i = 0;
+    function readUntilDelimit() {
+        let value = "";
+        while (i < str.length && str[i] !== delimit) {
+            value += str[i];
+            i++;
+        }
+        if (str[i] === delimit) i++;
+        return value;
+    }
+    function readNChars(n: number) {
+        let value = str.slice(i, i + n);
+        i += n;
+        return value;
+    }
+    while (i < str.length) {
+        let part = readUntilDelimit();
+        if (!part) {
+            let length = Number(readUntilDelimit());
+            if (Number.isNaN(length) || length < 0 || length > str.length) throw new Error(`Invalid length in nice array ${length}, array ${str}`);
+            let value = readNChars(length);
+            output.push(value);
+        } else {
+            output.push(part);
+        }
+    }
+    return output;
+}
+
+/** Just checks the pathPrefix, applying wildcard logic */
+function startsWithWildcard(authority: AuthorityPath, rootPath: string, childPath: string, onlyMatchChildren = false): boolean {
+    if (onlyMatchChildren && rootPath === childPath) return false;
+    if (!authority.emptyIsWildcard) return childPath.startsWith(rootPath);
+    // Eh... we might get lucky and they might both be wildcards?
+    if (childPath.startsWith(rootPath)) {
+        if (onlyMatchChildren) {
+            let rootParts = getPathFromStr(rootPath);
+            let childParts = getPathFromStr(childPath);
+            if (rootParts.length === childParts.length) return false;
+        }
+        return true;
+    }
+    // Because our wildcard is the shortest possible length, any match must be >= length,
+    //  so if childPath is shorter... it can't match.
+    if (childPath.length < rootPath.length) return false;
+
+    let rootParts = getPathFromStr(rootPath);
+    let childParts = getPathFromStr(childPath);
+
+
+    // If we are only matching children, and it isn't a child, don't mathc
+    if (onlyMatchChildren && rootParts.length === childParts.length) return false;
+
+    for (let i = 0; i < rootParts.length; i++) {
+        if (rootParts[i] === "") continue;
+        if (rootParts[i] !== childParts[i]) return false;
+    }
+    return true;
+}
+
+// TOOD: The routing hash (per depth) could be cached for free if we explicitly ran this in a loop. However...
+//      that would make per authority caching harder, so... I don't know how to take advantage of this.
+const isInAuthorityBase = measureWrap(function isInAuthorityBase(authority: AuthorityPath, path: string): boolean {
+    if (!startsWithWildcard(authority, authority.pathPrefix, path)) return false;
+    if (authority.excludedChildren) {
+        if (authority.excludedChildren.some(x => startsWithWildcard(authority, x, path, true))) return false;
+    }
+    if (authority.hash) {
+        let part = getPathIndex(path, authority.hash.depth);
+        if (part === undefined) return false;
+        // If we match a wildcard, then we definitely match
+        if (part !== "") {
+            if (!(matchesParentRangeFilterPart({ part, start: authority.hash.start, end: authority.hash.end }))) {
+                return false;
+            }
+        }
+    }
+    return true;
+});
+
+const isInAuthorityCache = cacheLimited(300, (authority: AuthorityPath) => {
+    return cacheLimited(10_000, (path: string) => isInAuthorityBase(authority, path));
+});
+export function isInAuthority(authority: AuthorityPath, path: string): boolean {
+    return isInAuthorityCache(authority)(path);
+}
+
+function getArchiveDirectory(authority: AuthorityPath): string {
+    let parts: string[] = [];
+    parts.push(authority.emptyIsWildcard ? "*" : "=");
+    parts.push(authority.pathPrefix);
+    if (authority.excludedChildren?.length || authority.hash) {
+        parts.push(String(authority.excludedChildren?.length ?? 0));
+        if (authority.excludedChildren) {
+            parts.push(...authority.excludedChildren);
+        }
+        if (authority.hash) {
+            parts.push(String(authority.hash.start));
+            parts.push(String(authority.hash.end));
+        }
+    }
+    return encodeNiceArray(parts, "-") + "/";
+}
+function decodeArchiveDirectory(directory: string): AuthorityPath {
+    if (!directory.includes("-")) {
+        return { pathPrefix: directory };
+    }
+    let parts = decodeNiceArray(directory, "-");
+    if (parts.length === 2) {
+        return { emptyIsWildcard: parts[0] === "*", pathPrefix: parts[1] };
+    }
+    if (parts.length < 3) throw new Error(`Invalid authority directory ${directory}, not enough parts. Expected at least 2, got ${parts.length}`);
+    let authority: AuthorityPath = {
+        emptyIsWildcard: parts.shift() === "*",
+        pathPrefix: parts.shift()!,
+    };
+    let excludedPrefixCount = Number(parts.shift());
+    if (excludedPrefixCount > 0) {
+        authority.excludedChildren = parts.splice(0, excludedPrefixCount);
+    }
+    if (parts.length > 0) {
+        authority.hash = {
+            depth: getPathDepth(authority.pathPrefix) + 1,
+            start: Number(parts.shift()),
+            end: Number(parts.shift()),
+        };
+    }
+    if (parts.length > 0) {
+        throw new Error(`Invalid authority directory ${directory}, too many parts. Have ${parts.length} too many.`);
+    }
+    return authority;
+}
+
+function gcd(a: number, b: number): number {
+    if (b === 0) return a;
+    return gcd(b, a % b);
+}
+
+function modsOverlap(a: { mod: number, value: number }, b: { mod: number, value: number }) {
+    // Basically... if a and b are unrelated, the mod becomes 1, and so the values are equal
+    //  (so they always overlap).
+    // Otherwise, we might be able to tell overlap vs not overlap. Ex, for 6 and 10. Despite not being
+    //      factors of each other, they are both even, and so if we know if the mod is even the original number
+    //      was even, etc. So if they differ in their value mod 2, then they don't overlap.
+    let baseMod = gcd(a.mod, b.mod);
+    let aValue = a.value % baseMod;
+    let bValue = b.value % baseMod;
+    return aValue === bValue;
+}
+
+/** Due to file max lengths the directory might not really match. It might also only partially overlap, due
+ *      to range encodings.
+ *      - We don't match ancestors, as AuthorityPaths never never match ancestors.
+ */
+function mightMatchArchiveDirectory(authority: AuthorityPath, directory: string): boolean {
+    let dirAuthority: AuthorityPath;
+    try {
+        dirAuthority = decodeArchiveDirectory(directory);
+    } catch {
+        // Hack, to support legacy databases (one we merge archives files we should be able to
+        //  run merge on all to update the paths, and remove this try/catch)
+        return true;
+    }
+    return authoritiesMightOverlap(authority, dirAuthority);
+}
+function authoritiesMightOverlap(other: AuthorityPath, current: AuthorityPath): boolean {
+
+    let otherNoHash = { ...other, hashes: undefined };
+    let currentNoHash = { ...current, hashes: undefined };
+    let matchesWithoutHash = isInAuthority(otherNoHash, currentNoHash.pathPrefix) || isInAuthority(currentNoHash, otherNoHash.pathPrefix);
+    // If we ignore hash and we don't match then we definitely don't match
+    if (!matchesWithoutHash) {
+        return false;
+    }
+    // If neither have hashes, then we match (as we discarded the not match case above)
+    if (!other.hash && !current.hash) return true;
+
+    let parent = other.pathPrefix.length > current.pathPrefix.length ? current : other;
+    let child = other.pathPrefix.length > current.pathPrefix.length ? other : current;
+    // NOTE: As we match in one direction without the hash, it means our path prefixes DEFINITELY match,
+    //  so we don't need to check them again.
+    if (!parent.hash && child.hash) {
+        return true;
+    } else if (parent.hash && !child.hash) {
+        return getPathDepth(child.pathPrefix) === parent.hash.depth;
+    } else {
+        // Both have hashes
+
+        // If the prefix doesn't match, we don't match
+        if (otherNoHash.pathPrefix !== currentNoHash.pathPrefix) return false;
+
+        if (otherNoHash.hashes && currentNoHash.hashes) {
+            return modsOverlap(otherNoHash.hashes, currentNoHash.hashes);
+        }
+    }
+
+    return false;
+}
+
+
+
+
+let onReadyReady = (nodeId: string, time: number) => { };
+class PathControllerBase {
+    public async getAuthorityPaths() {
+        return pathValueAuthority2.getSelfAuthorities();
+    }
+
+    public async getCreateTime() {
+        return pathValueAuthority2.createTime;
+    }
+
+    public async isReadReady() {
+        return pathValueAuthority2.isSelfReadReady();
+    }
+
+    public async broadcastReadReady(time: number) {
+        onReadyReady(IdentityController_getCurrentReconnectNodeIdAssert(), time);
+    }
+}
+const PathController = SocketFunction.register(
+    "PathController-64dae2b5-c719-423a-b547-64555b26f942",
+    new PathControllerBase(),
+    () => ({
+        getAuthorityPaths: {},
+        getCreateTime: {},
+        isReadReady: {
+            // NOTE: Avoid using client hooks, so we don't change our identity, which
+            //  can be overly slow when the node is likely disconnected anyways.
+            noClientHooks: true,
+        },
+        broadcastReadReady: {},
+    })
+);
+export async function debug_isReadReady(nodeId: string) {
+    return PathController.nodes[nodeId].isReadReady();
+}
+export async function debug_getAuthorityPaths(nodeId: string) {
+    return PathController.nodes[nodeId].getAuthorityPaths();
+}
+
+
+export const pathValueAuthority2 = new NodePathAuthorities();
+export const nodePathAuthority = pathValueAuthority2;