querysub 0.406.0 → 0.408.0

package/src/0-path-value-core/PathRouter.ts

@@ -9,6 +9,7 @@ import { unique } from "../misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
 import { sha256 } from "js-sha256";
+import { rangesOverlap, removeRange } from "../rangeMath";
 
 
 // Cases
@@ -36,23 +37,35 @@ export type AuthoritySpec = {
     excludeDefault?: boolean;
 };
 
+export function debugSpec(spec: AuthoritySpec) {
+    return {
+        info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,
+        spec: { do: { not: { expand: { spec } } } },
+    };
+}
+
+function getMatchingPrefix(spec: AuthoritySpec, path: string): string | undefined {
+    let longestPrefix: string | undefined;
+    for (let prefix of spec.prefixes) {
+        if (path.startsWith(prefix) && prefix !== path) {
+            if (!longestPrefix || longestPrefix.length < prefix.length) {
+                longestPrefix = prefix;
+            }
+        }
+    }
+    return longestPrefix;
+}
+
 
 export class PathRouter {
 
     public static async waitUntilReady() {
         await authorityLookup.startSyncing();
     }
-    /** NOTE: Parent watches are a little bit special. If it's a parent watch, we always hash it, assuming the parent is a prefix. And as most of the watches are parent watches, we're usually going to do this, and so it actually is independent of the topology. 
-        - The topology is really only used for the initial sync, which will use matchesAuthoritySpec, which gets the full routing value, AND, for disk storage.
-     */
-    @measureFnc
-    public static getRouteChildKey(path: string): number {
-        let key = getLastPathPart(path);
-        return this.getSingleKeyRoute(key);
-    }
+
     // NOTE: For non-prefix values, breaking up by routes on the file system becomes complicated, and so we just all non-prefix values in the same file. However, in memory, in some places, we need route values for every single path, such as for FunctionRunner, so it can distribute the function running evenly, without overlap.
     @measureFnc
-    private static getRouteFull(config: {
+    public static getRouteFull(config: {
         path: string;
         spec: AuthoritySpec;
     }): number {
@@ -61,22 +74,38 @@ export class PathRouter {
         path = hack_stripPackedPath(path);
         let override = getRoutingOverride(path);
         if (override) {
-            if (!hasPrefixHash({ spec, prefixHash: override.prefixHash })) return -1;
+            if (spec.excludeDefault && !hasPrefixHash({ spec, prefixHash: override.prefixHash })) return -1;
+            if (override.route < spec.routeStart || override.route >= spec.routeEnd) return -1;
             return override.route;
         }
 
-        let prefix = spec.prefixes.find(x => path.startsWith(x) && x !== path);
+        let prefix = getMatchingPrefix(spec, path);
         if (prefix) {
             let key = getPathIndex(path, getPathDepth(prefix));
             if (key === undefined) {
+                require("debugbreak")(2);
+                debugger;
                 throw new Error(`Impossible, hash index ${getPathDepth(prefix)} is out of range for path ${path}, but it matched the prefix ${prefix}`);
             }
-            return this.getSingleKeyRoute(key);
+            let route = this.getSingleKeyRoute(key);
+            if (route < spec.routeStart || route >= spec.routeEnd) return -1;
+            return route;
         }
         if (spec.excludeDefault) return -1;
-        let hash = this.getSingleKeyRoute(path);
-        if (hash < spec.routeStart || hash >= spec.routeEnd) return -1;
-        return hash;
+        let route = this.getSingleKeyRoute(path);
+        if (route < spec.routeStart || route >= spec.routeEnd) return -1;
+        return route;
+    }
+
+    // Mostly for debugging
+    @measureFnc
+    public static getAllRoutes(path: string): number[] {
+        let routes: number[] = [];
+        for (let authority of authorityLookup.getTopologySync()) {
+            let route = this.getRouteFull({ path, spec: authority.authoritySpec });
+            routes.push(route);
+        }
+        return routes;
     }
 
     private static lastKeyRoute = {
@@ -86,6 +115,12 @@ export class PathRouter {
     // Takes a key which is a part of a path. Mostly used PathRouterRouteOverride, or other PathRouter helpers.
     public static getSingleKeyRoute(key: string): number {
         if (key && this.lastKeyRoute.key === key) return this.lastKeyRoute.route;
+        let override = getRoutingOverride(key);
+        if (override) {
+            this.lastKeyRoute.key = key;
+            this.lastKeyRoute.route = override.route;
+            return override.route;
+        }
         let hash = fastHash(key);
         let route = hash % (1000 * 1000 * 1000) / (1000 * 1000 * 1000);
         this.lastKeyRoute.key = key;
@@ -104,6 +139,9 @@ export class PathRouter {
     private static getPrefixHash(prefix: string): string {
         return Buffer.from(sha256(prefix), "hex").toString("base64").slice(0, 6);
     }
+    private static isPrefixHash(hash: string): boolean {
+        return hash.length === 6 && /^[a-zA-Z0-9]+$/.test(hash);
+    }
     private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
@@ -125,7 +163,7 @@ export class PathRouter {
         return {
             rangeStart: parseFloat(parts[1]),
             rangeEnd: parseFloat(parts[2]),
-            prefixHashes: parts.slice(3),
+            prefixHashes: parts.slice(3).filter(this.isPrefixHash),
         };
     }
 
@@ -137,26 +175,19 @@ export class PathRouter {
         // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
         const MAX_PREFIXES_PER_FILE = 50;
-        const PREFIX_COVER_FRACTION = 0.95;
-        const TARGET_VALUES_PER_FILE = 50 * 1000;
-        if (values.length < TARGET_VALUES_PER_FILE) {
-            return new Map([[this.encodeIdentifier("remaining"), values]]);
-        }
+        const PREFIX_COVER_FRACTION = 0.99;
+        const TARGET_VALUES_PER_SHARD_GROUP = 10 * 1000 * 1000;
+        const TARGET_SHARD_SIZE = 50 * 1000;
+        const MIN_SHARD_FILE_COUNT = 10;
+        const SHARD_THRESHOLD = 1000;
 
         let prefixes = ourSpec.prefixes.slice();
         sort(prefixes, x => x.length);
-        function getPrefix(path: string): string | undefined {
-            for (let prefix of prefixes) {
-                if (path.startsWith(prefix) && prefix !== path) return prefix;
-            }
-            return undefined;
-        }
-
 
         // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash. 
         let byPrefix = new Map<string | undefined, PathValue[]>();
         for (let value of values) {
-            let prefix = getPrefix(value.path);
+            let prefix = getMatchingPrefix(ourSpec, value.path);
             let values = byPrefix.get(prefix);
             if (!values) {
                 values = [];
@@ -194,7 +225,7 @@ export class PathRouter {
             }
             let last = groups[groups.length - 1];
             if (
-                last.count > 0 && last.count + prefixGroup.values.length > TARGET_VALUES_PER_FILE
+                last.count > 0 && last.count + prefixGroup.values.length > TARGET_VALUES_PER_SHARD_GROUP
                 || last.prefixes.length >= MAX_PREFIXES_PER_FILE
             ) {
                 groups.push({
@@ -210,42 +241,44 @@ export class PathRouter {
             prefixLeft -= prefixGroup.values.length;
         }
 
-
         let finalFiles = new Map<string, PathValue[]>();
         for (let group of groups) {
-            if (group.prefixes.length === 1 && group.count > TARGET_VALUES_PER_FILE) {
-                // Split by routing hash
-                let values = group.values.flat();
-                let splitCount = Math.ceil(values.length / TARGET_VALUES_PER_FILE);
-                let byRouteGroup = new Map<number, PathValue[]>();
-                let prefix = group.prefixes[0];
-                let hashIndex = getPathDepth(prefix);
-                for (let value of values) {
-                    let key = getPathIndex(value.path, hashIndex);
-                    if (key === undefined) {
-                        throw new Error(`Impossible, hash index ${hashIndex} is out of range for path ${value.path}, but it matched the prefix ${prefix}`);
-                    }
-                    let route = this.getSingleKeyRoute(key);
-                    let routeIndex = Math.floor(route * splitCount);
-                    let routeValues = byRouteGroup.get(routeIndex);
-                    if (!routeValues) {
-                        routeValues = [];
-                        byRouteGroup.set(routeIndex, routeValues);
-                    }
-                    routeValues.push(value);
-                }
-                for (let [routeIndex, routeValues] of byRouteGroup) {
-                    let rangeStart = routeIndex / splitCount;
-                    let rangeEnd = (routeIndex + 1) / splitCount;
-                    let identifier = this.encodeIdentifier({ prefixes: [prefix], rangeStart, rangeEnd });
-                    finalFiles.set(identifier, routeValues);
-                }
-            } else {
+            if (group.count < SHARD_THRESHOLD) {
                 let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart: 0, rangeEnd: 1 });
                 finalFiles.set(identifier, group.values.flat());
+                continue;
+            }
+            // Split by routing hash
+            let values = group.values.flat();
+            let splitCount = Math.max(MIN_SHARD_FILE_COUNT, Math.ceil(values.length / TARGET_SHARD_SIZE));
+            let byRouteGroup = new Map<number, PathValue[]>();
+            for (let value of values) {
+                let route = this.getRouteFull({
+                    path: value.path,
+                    spec: {
+                        nodeId: "",
+                        prefixes: group.prefixes,
+                        routeStart: 0,
+                        routeEnd: 1,
+                    }
+                });
+                let routeIndex = Math.floor(route * splitCount);
+                let routeValues = byRouteGroup.get(routeIndex);
+                if (!routeValues) {
+                    routeValues = [];
+                    byRouteGroup.set(routeIndex, routeValues);
+                }
+                routeValues.push(value);
+            }
+            for (let [routeIndex, routeValues] of byRouteGroup) {
+                let rangeStart = routeIndex / splitCount;
+                let rangeEnd = (routeIndex + 1) / splitCount;
+                let identifier = this.encodeIdentifier({ prefixes: group.prefixes, rangeStart, rangeEnd });
+                finalFiles.set(identifier, routeValues);
             }
         }
 
+        // NOTE: There could be a huge number of prefixes and we can't pack them all into one file because of the prefix limit, so this will write any remaining values. 
         if (remainingValues.length > 0) {
             let identifier = this.encodeIdentifier("remaining");
             finalFiles.set(identifier, remainingValues.flat());
@@ -268,6 +301,46 @@ export class PathRouter {
         return decodeObj.rangeStart < authority.routeEnd && decodeObj.rangeEnd > authority.routeStart;
     }
 
+    @measureFnc
+    public static overlapsAuthority(authority1: AuthoritySpec, authority2: AuthoritySpec): boolean {
+        // TODO: This becomes complicated because of exclude default, although I feel like there has to be a way to simplify it? Eh... whatever.
+
+        // Normalize it so if only one excludes default, it's always going to be the second one. 
+        if (authority1.excludeDefault && !authority2.excludeDefault) return this.overlapsAuthority(authority2, authority1);
+
+        let doRangesOverlap = rangesOverlap({ start: authority1.routeStart, end: authority1.routeEnd }, { start: authority2.routeStart, end: authority2.routeEnd });
+
+        // If their prefixes are identical, then it's purely a range check
+        if (authority1.prefixes.length === authority2.prefixes.length && authority1.prefixes.every(x => authority2.prefixes.includes(x))) {
+            return doRangesOverlap;
+        }
+
+        // If they have any prefixes which are identical and the ranges overlap, then they overlap.
+        if (doRangesOverlap) {
+            if (authority1.prefixes.some(x => authority2.prefixes.includes(x))) {
+                return true;
+            }
+        }
+        // If any of their prefixes are under the prefix and match it, then that's a match. 
+        if (authority1.prefixes.some(x => this.matchesAuthoritySpec(authority2, x))) {
+            return true;
+        }
+        if (authority2.prefixes.some(x => this.matchesAuthoritySpec(authority1, x))) {
+            return true;
+        }
+        if (authority1.excludeDefault && authority2.excludeDefault) {
+            // No shared prefixes, and none of them are nested under each other, and we don't include defaults, so neither match. 
+            return false;
+        }
+        // If their prefixes are entirely unrelated, it means they're going to hash differently, so they do overlap. 
+        return true;
+    }
+    @measureFnc
+    public static getAllOverlappingAuthorities(authority: AuthoritySpec): AuthoritySpec[] {
+        let allAuthorities = authorityLookup.getTopologySync();
+        return allAuthorities.filter(x => this.overlapsAuthority(authority, x.authoritySpec)).map(x => x.authoritySpec);
+    }
+
 
 
     public static isLocalPath(path: string): boolean {
@@ -283,11 +356,12 @@ export class PathRouter {
 
 
     // NOTE: This might return overlapping specs. Presently, this is only used when we're loading our initial data, so it's fine. However, if we use this for another purpose in the future, it might cause problems. So we might need to implement a different function for that theoretical future purpose.
+    // NOTE: Only takes remote nodes as presently this is just used during startup.
     @measureFnc
     public static getAuthoritySources(config: {
         target: AuthoritySpec;
         preferredNodeIds?: string[];
-    }): (AuthoritySpec & { useFullPathHash?: boolean })[] {
+    }): AuthoritySpec[] {
         let { target } = config;
         let allSources = authorityLookup.getTopologySync();
         allSources = allSources.filter(x => !isOwnNodeId(x.nodeId));
@@ -368,41 +442,21 @@ export class PathRouter {
             for (let source of group) {
                 let s = source.routeStart;
                 let e = source.routeEnd;
-                // Find if it overlaps any missing range, and if so, cut out that part of the missing change, and add the nodeId to usedNodeIds
-                for (let i = missingRanges.length - 1; i >= 0; i--) {
-                    let missingRange = missingRanges[i];
-                    if (s >= missingRange.end || e <= missingRange.start) continue;
-                    let startTaken = Math.max(missingRange.start, s);
-                    let endTaken = Math.min(missingRange.end, e);
-                    // NOTE: It's not ideal that we might have to fragment one node ID between multiple requests. However, in practice, there shouldn't be much fragmentation here. The ranges that our nodes are breaking down by should be consistent, so there's actually no overlap or subsets.
+                let { removedRanges } = removeRange(missingRanges, { start: s, end: e });
+                for (let removedRange of removedRanges) {
                     usedParts.push({
                         nodeId: source.nodeId,
-                        routeStart: startTaken,
-                        routeEnd: endTaken,
+                        routeStart: removedRange.start,
+                        routeEnd: removedRange.end,
                         prefixes: target.prefixes,
                         excludeDefault: target.excludeDefault,
-                        useFullPathHash,
                     });
-                    missingRanges.splice(i, 1);
-                    // Add back the parts we didn't overlap
-                    if (missingRange.start < s) {
-                        missingRanges.push({
-                            start: missingRange.start,
-                            end: s,
-                        });
-                    }
-                    if (missingRange.end > e) {
-                        missingRanges.push({
-                            start: e,
-                            end: missingRange.end,
-                        });
-                    }
                 }
                 if (missingRanges.length === 0) break;
             }
             if (missingRanges.length === 0) {
                 if (!hashesSameAsTarget(group[0])) {
-                    console.warn(`Found a cohesive group that doesn't hash the same way we do. Expanding our range to be the full range of the group. This will be slower than it needs to.`);
+                    console.warn(`Could only match a cohesive group that doesn't hash the same way we do. Expanding our range to be the full range of the group. This will be slower than it needs to, and might cause issues that result in only partial synchronization. This should be fixable with a deploy.`, config.target);
                 }
                 return usedParts;
             }
@@ -414,67 +468,141 @@ export class PathRouter {
     }
 
 
+    // NOTE: The returned nodes are guaranteed to hash in the same way (either all child key hashing for the path children, or all path hashing). This is required, otherwise it would mean if you take a single child path, It might have two different routing values depending on which node it matches, which means even if those ranges don't overlap, different routing values mean it could match two ranges, which is impossible and would break things (and it would also mean there would be values that wouldn't match anything, which I guess is even worse).
     @measureFnc
     public static getChildReadNodes(path: string, config?: {
         preferredNodeIds?: string[];
+        onlyOwnNodes?: boolean;
     }): {
-        // By default we hash the key directly under the path. However, If that is not how our nodes were sharded, as in they didn't use it as a prefix, then they will be sharded by hashing the full path, and so when we receive the data, we need to keep this in mind and filter the data based on the full path hash.
-        useFullPathHash?: boolean;
-
         // NOTE: If at all possible, we will cover all ranges. Node of the returned nodes will be redundant.
         //  - Sorted by range.start
         nodes: {
             nodeId: string;
+            authoritySpec: AuthoritySpec;
             // The range of hashes this node owns, for the child keys of path
             //  (If the node doesn't restrict the range, it will just be { start: 0, end: 1 })
             range: { start: number; end: number };
         }[];
     } {
-        if (this.isSelfAuthority(path)) {
-            return { nodes: [{ nodeId: getOwnNodeId(), range: { start: 0, end: 1 } }] };
-        }
+        let preferredNodeIds = new Set(config?.preferredNodeIds ?? []);
 
         // If a prefix is a parent of path, then it is the same as matching just the path directly
         // (If our prefix directly equals one of the other matches, then it's more complicated, As then, the child keys of path are what is hashed, and so all the children will have different routes, so we might match multiple nodes. The same thing if we're matching the remaining case, in which case it's a full path hash, so the child key matters, and again, different routes).
         //      - The different route case is how the FuntionRunner works, and without it large databases couldn't run functions. However, most applications won't directly use it.
-        let allSources = authorityLookup.getTopologySync();
-        allSources = allSources.filter(x => !isOwnNodeId(x.nodeId));
-        let nestedMatches = allSources.filter(x =>
-            x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
-            && this.matchesAuthoritySpec(x.authoritySpec, path)
-        );
+        let allSources = config?.onlyOwnNodes ? [{ nodeId: getOwnNodeId(), authoritySpec: authorityLookup.getOurSpec() }] : authorityLookup.getTopologySync();
+        // Prefer our own node
+        sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
+
+
+        // Direct prefixes always take priority, as almost everything is under a prefix anyways...
+
+
+        // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
+        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y === path)).map(x => x.authoritySpec);
+        if (hasPrefix.length > 0) {
+            shuffle(hasPrefix, Math.random());
+            sort(hasPrefix, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
+
+            let missingRanges: { start: number; end: number }[] = [{
+                start: 0,
+                end: 1,
+            }];
+            let usedParts: {
+                nodeId: string;
+                authoritySpec: AuthoritySpec;
+                range: { start: number; end: number };
+            }[] = [];
+            for (let source of hasPrefix) {
+                let s = source.routeStart;
+                let e = source.routeEnd;
+                let { removedRanges } = removeRange(missingRanges, { start: s, end: e });
+                // NOTE: It's not ideal that we might have to fragment one node ID between multiple requests. However, in practice, there shouldn't be much fragmentation here. The ranges that our nodes are breaking down by should be consistent, so there's actually no overlap or subsets.
+                for (let removedRange of removedRanges) {
+                    usedParts.push({
+                        nodeId: source.nodeId,
+                        range: removedRange,
+                        authoritySpec: source,
+                    });
+                }
+                if (missingRanges.length === 0) break;
+            }
+            if (missingRanges.length === 0) {
+                return { nodes: usedParts };
+            }
+        }
+
+        let nestedMatches = allSources.filter(x => {
+            // There's nested prefixes, so if we match any prefix explicitly, we can't just take one of the previous prefixes because that isn't how the hashing will work. 
+            // - This happens if it's a direct match, but one of the shards is down, in which case we can't get a full match.
+            if (x.authoritySpec.prefixes.some(y => y === path)) return false;
+
+            // If our path, which we're going to read the children of, is the child of another path, then it means in that other path, the child key will be known to us constant, and so we're going to match exactly one authority. 
+            return (
+                x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
+                && this.matchesAuthoritySpec(x.authoritySpec, path)
+            );
+        });
         if (nestedMatches.length > 0) {
             shuffle(nestedMatches, Math.random());
-            let preferredNodeIds = new Set(config?.preferredNodeIds ?? []);
             sort(nestedMatches, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
+            sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
             return {
-                nodes: nestedMatches.map(x => ({
+                // Only need to take the first match. Our path is picked by the prefix, and the prefix only hashes the direct child, and we're more deeply nested than that, which means... the route for all of our children will be identical, so this node matches all of our children. 
+                nodes: nestedMatches.slice(0, 1).map(x => ({
                     nodeId: x.nodeId,
-                    // NOTE: Our path is picked by the prefix, and the prefix only hashes the direct child, and we're more deeply nested than that, which means... the route for all of our children will be identical, so this node matches all of our children. 
+                    authoritySpec: x.authoritySpec,
                     range: { start: 0, end: 1 },
                 })),
             };
         }
 
-        let fullSources = this.getAuthoritySources({
-            target: {
-                nodeId: "",
-                prefixes: [path],
-                routeStart: 0,
-                routeEnd: 1,
-                excludeDefault: true,
-            },
-            preferredNodeIds: config?.preferredNodeIds,
+        // If we are not under any prefixes of it, then it will be a full path hash
+        let fullPathMatches = allSources.filter(x => {
+            return !x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path) && !x.authoritySpec.excludeDefault;
         });
-        if (fullSources.length === 0) return { nodes: [] };
+        // Same as prefix matches. Not preferred, and not preferred over being under a prefix, but required for some root data, or data with no prefixes.
+        if (fullPathMatches.length > 0) {
+            shuffle(fullPathMatches, Math.random());
+            sort(fullPathMatches, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
+            sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
+            let missingRanges: { start: number; end: number }[] = [{
+                start: 0,
+                end: 1,
+            }];
+            let usedParts: {
+                nodeId: string;
+                authoritySpec: AuthoritySpec;
+                range: { start: number; end: number };
+            }[] = [];
+            for (let source of fullPathMatches) {
+                let s = source.authoritySpec.routeStart;
+                let e = source.authoritySpec.routeEnd;
+                let { removedRanges } = removeRange(missingRanges, { start: s, end: e });
+                // NOTE: It's not ideal that we might have to fragment one node ID between multiple requests. However, in practice, there shouldn't be much fragmentation here. The ranges that our nodes are breaking down by should be consistent, so there's actually no overlap or subsets.
+                for (let removedRange of removedRanges) {
+                    usedParts.push({
+                        nodeId: source.nodeId,
+                        authoritySpec: source.authoritySpec,
+                        range: removedRange,
+                    });
+                }
+                if (missingRanges.length === 0) break;
+            }
+            if (missingRanges.length === 0) {
+                return { nodes: usedParts };
+            }
+        }
 
-        return {
-            useFullPathHash: fullSources.some(x => x.useFullPathHash),
-            nodes: fullSources.map(x => ({
-                nodeId: x.nodeId,
-                range: { start: x.routeStart, end: x.routeEnd },
-            })),
-        };
+
+
+
+        if (!config?.onlyOwnNodes) {
+            // TODO: We could maybe match a partial match. However, even that is suspect. The site being partially broken is almost worse than it being completely broken. We should just get ALL the shards running again...
+
+            // NOTE: We *could* actually synchronize it even if it doesn't have a prefix shard as we can fall back to just the full path sharding. However, it becomes very complicated if we want a specific range, and then it becomes complicated if it then switches to prefix hashing (With the nodes that were using the full path hashing slowly going away). AND... key synchronization IS slow, so it's good to discourage it in general. 
+            console.error(`Want to sync a prefix which is not under an existing prefix, nor equal to a prefix. 1) The servers are down. 2) Don't access the .keys() 3) call addRoutingPrefixForDeploy to add a route/parent route explicitly (as is done in PathFunctionRunner.ts). Path: ${JSON.stringify(path)}`, { path, allSources });
+        }
+        return { nodes: [] };
     }
 
 
@@ -538,3 +666,5 @@ export class PathRouter {
 
 }
 
+
+(globalThis as any).PathRouter = PathRouter;

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -22,7 +22,7 @@ let keySpecialIdentifier = (
 export function createRoutingOverrideKey(config: {
     originalKey: string;
     routeKey: string;
-    // This is not the prefix that it has to match. This is the prefix we remap it to. And so anything which matches this remap prefix will match this routing key. 
+    // This is the prefix it has the equivalent of. We need this, so if something excludes default, it doesn't automatically get every routing overridden value.
     remappedPrefix: string;
 }) {
     let { originalKey, routeKey, remappedPrefix } = config;

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -55,13 +55,15 @@ export function getEmptyAuthoritySpec(): AuthoritySpec {
         routeStart: -1,
         routeEnd: -1,
         prefixes: [],
+        excludeDefault: true,
     };
 }
-export function getAllAuthoritySpec(): AuthoritySpec {
+export async function getAllAuthoritySpec(): Promise<AuthoritySpec> {
+    let prefixes = await getShardPrefixes();
     return {
         nodeId: "",
         routeStart: 0,
         routeEnd: 1,
-        prefixes: [],
+        prefixes: prefixes,
     };
 }