querysub 0.424.0 → 0.426.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.424.0",
+  "version": "0.426.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/AuthorityLookup.ts

@@ -12,7 +12,7 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isClient } from "../config2";
 import { getPathStr1 } from "../path";
 import { timeoutToError } from "../errors";
-import { AuthoritySpec } from "./PathRouter";
+import { AuthoritySpec, parsePrefixMatcher } from "./PathRouter";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getAllAuthoritySpec, getEmptyAuthoritySpec } from "./PathRouterServerAuthoritySpec";
 
@@ -208,10 +208,10 @@ class AuthorityLookup {
             //  - Get all node IDs should restrict our nodes to just the browser node ID. If we ever change this, then either it's redundant nodes and they all have all the same data, or we need to figure out what data they have, And as their proxies, it probably won't be their actual authority data. So that will require new API functions, etc. 
             await new Promise(r => setImmediate(r));
             await delay(1);
-            let { getPrefixesForDeploy } = await import("../3-path-functions/syncSchema");
+            let prefixes = await this.getPrefixesForDeploy();
             this.updatePaths(nodeId, {
                 nodeId: nodeId,
-                prefixes: await getPrefixesForDeploy(),
+                prefixes: prefixes.map(p => parsePrefixMatcher(p)),
                 routeStart: 0,
                 routeEnd: 1,
             }, true);
@@ -233,6 +233,11 @@ class AuthorityLookup {
         this.updatePaths(nodeId, otherPaths.spec, otherPaths.isReady);
         this.disconnectWatch(nodeId);
     }
+
+    private async getPrefixesForDeploy() {
+        let { getPrefixesForDeploy } = await import("../3-path-functions/syncSchema");
+        return await getPrefixesForDeploy();
+    }
 }
 export const authorityLookup = new AuthorityLookup();
 (globalThis as any).authorityLookup = authorityLookup;

package/src/0-path-value-core/PathRouter.ts

@@ -1,5 +1,5 @@
 import { sort } from "socket-function/src/misc";
-import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1, hack_stripPackedPath } from "../path";
+import { getLastPathPart, getPathDepth, getPathFromStr, getPathIndex, getPathStr, getPathStr1, hack_stripPackedPath } from "../path";
 import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
@@ -33,10 +33,65 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
-    prefixes: string[];
+    prefixes: PrefixMatcher[];
     excludeDefault?: boolean;
 };
 
+
+export type PrefixMatcher = {
+    prefix: string;
+    additionalMatches: {
+        value: string;
+        index: number;
+    }[];
+    childKeyIndex: number;
+    originalPrefix: string;
+    prefixPartLength: number;
+};
+function matchesPrefix(matcher: PrefixMatcher, path: string): boolean {
+    if (!path.startsWith(matcher.prefix)) return false;
+    if (matcher.prefix === path) return false;
+    if (getPathIndex(path, matcher.childKeyIndex) === undefined) return false;
+    for (let additionalMatch of matcher.additionalMatches) {
+        if (getPathIndex(path, additionalMatch.index) !== additionalMatch.value) {
+            return false;
+        }
+    }
+    return true;
+}
+/** Empty string path part becomes a wildcard */
+export function parsePrefixMatcher(prefixPath: string): PrefixMatcher {
+    // So up until the first wild card, that's a prefix. But then after that, we then need to skip all the wild cards, of course. And then for the non-wild cards, add them as additional matches. And then obviously set the child key index to just be the part's length. 
+    let parts = getPathFromStr(prefixPath);
+    let wildcardIndex = parts.indexOf("");
+    if (wildcardIndex === -1) {
+        return {
+            prefix: prefixPath,
+            additionalMatches: [],
+            childKeyIndex: parts.length,
+            originalPrefix: prefixPath,
+            prefixPartLength: parts.length,
+        };
+    }
+    let prefix = getPathStr(parts.slice(0, wildcardIndex));
+    let additionalMatches: { value: string; index: number }[] = [];
+    for (let i = wildcardIndex + 1; i < parts.length; i++) {
+        let part = parts[i];
+        if (part === "") continue;
+        additionalMatches.push({
+            value: part,
+            index: i,
+        });
+    }
+    return {
+        prefix,
+        additionalMatches,
+        childKeyIndex: wildcardIndex,
+        originalPrefix: prefixPath,
+        prefixPartLength: parts.length,
+    };
+}
+
 export function debugSpec(spec: AuthoritySpec) {
     return {
         info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,
@@ -44,16 +99,16 @@ export function debugSpec(spec: AuthoritySpec) {
     };
 }
 
-function getMatchingPrefix(spec: AuthoritySpec, path: string): string | undefined {
-    let longestPrefix: string | undefined;
-    for (let prefix of spec.prefixes) {
-        if (path.startsWith(prefix) && prefix !== path) {
-            if (!longestPrefix || longestPrefix.length < prefix.length) {
-                longestPrefix = prefix;
+function getMatchingPrefix(spec: AuthoritySpec, path: string): PrefixMatcher | undefined {
+    let longestMatcher: PrefixMatcher | undefined;
+    for (let matcher of spec.prefixes) {
+        if (matchesPrefix(matcher, path)) {
+            if (!longestMatcher || longestMatcher.prefixPartLength < matcher.prefixPartLength) {
+                longestMatcher = matcher;
             }
         }
     }
-    return longestPrefix;
+    return longestMatcher;
 }
 
 
@@ -81,11 +136,11 @@ export class PathRouter {
 
         let prefix = getMatchingPrefix(spec, path);
         if (prefix) {
-            let key = getPathIndex(path, getPathDepth(prefix));
+            let key = getPathIndex(path, prefix.childKeyIndex);
             if (key === undefined) {
                 require("debugbreak")(2);
                 debugger;
-                throw new Error(`Impossible, hash index ${getPathDepth(prefix)} is out of range for path ${path}, but it matched the prefix ${prefix}`);
+                throw new Error(`Impossible, hash index ${prefix.childKeyIndex} is out of range for path ${path}, but it matched the prefix ${prefix.originalPrefix}`);
             }
             let route = this.getSingleKeyRoute(key);
             if (route < spec.routeStart || route >= spec.routeEnd) return -1;
@@ -136,13 +191,13 @@ export class PathRouter {
     }
 
 
-    private static getPrefixHash(prefix: string): string {
-        return Buffer.from(sha256(prefix), "hex").toString("base64").slice(0, 6);
+    private static getPrefixHash(prefix: PrefixMatcher): string {
+        return Buffer.from(sha256(prefix.originalPrefix), "hex").toString("base64").slice(0, 6);
     }
     private static isPrefixHash(hash: string): boolean {
         return hash.length === 6 && /^[a-zA-Z0-9]+$/.test(hash);
     }
-    private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
+    private static encodeIdentifier(config: { prefixes: PrefixMatcher[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
         return [
@@ -151,7 +206,7 @@ export class PathRouter {
             rangeEnd.toString(),
             ...prefixes.map(x => this.getPrefixHash(x)),
             // Add some prefixes. They'll be interpreted as hashes, but it's fine because it's highly unlikely they'll collide with a real hash. AND... It's very useful for debugging what's in which file. 
-            ...prefixes.slice(0, 5).map(x => getLastPathPart(x).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
+            ...prefixes.slice(0, 5).map(x => getLastPathPart(x.originalPrefix).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
             ""
         ].join("!");
 
@@ -182,10 +237,10 @@ export class PathRouter {
         const SHARD_THRESHOLD = 1000;
 
         let prefixes = ourSpec.prefixes.slice();
-        sort(prefixes, x => x.length);
+        sort(prefixes, x => x.prefixPartLength);
 
         // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash. 
-        let byPrefix = new Map<string | undefined, PathValue[]>();
+        let byPrefix = new Map<PrefixMatcher | undefined, PathValue[]>();
         for (let value of values) {
             let prefix = getMatchingPrefix(ourSpec, value.path);
             let values = byPrefix.get(prefix);
@@ -203,7 +258,7 @@ export class PathRouter {
         sort(prefixGroups, x => -x.values.length);
 
         let groups: {
-            prefixes: string[];
+            prefixes: PrefixMatcher[];
             values: PathValue[][];
             count: number;
         }[] = [];
@@ -305,34 +360,30 @@ export class PathRouter {
     public static overlapsAuthority(authority1: AuthoritySpec, authority2: AuthoritySpec): boolean {
         // TODO: This becomes complicated because of exclude default, although I feel like there has to be a way to simplify it? Eh... whatever.
 
-        // Normalize it so if only one excludes default, it's always going to be the second one. 
-        if (authority1.excludeDefault && !authority2.excludeDefault) return this.overlapsAuthority(authority2, authority1);
-
         let doRangesOverlap = rangesOverlap({ start: authority1.routeStart, end: authority1.routeEnd }, { start: authority2.routeStart, end: authority2.routeEnd });
 
         // If their prefixes are identical, then it's purely a range check
-        if (authority1.prefixes.length === authority2.prefixes.length && authority1.prefixes.every(x => authority2.prefixes.includes(x))) {
+        if (authority1.prefixes.length === authority2.prefixes.length && authority1.prefixes.every(x => authority2.prefixes.some(y => x.originalPrefix === y.originalPrefix))) {
             return doRangesOverlap;
         }
 
-        // If they have any prefixes which are identical and the ranges overlap, then they overlap.
-        if (doRangesOverlap) {
-            if (authority1.prefixes.some(x => authority2.prefixes.includes(x))) {
-                return true;
+        // If one has excludeDefault and is a subset of the other's prefixes, then it's a range check
+        if (authority1.excludeDefault) {
+            let authority2Prefixes = new Set(authority2.prefixes.map(x => x.originalPrefix));
+            let isSubset = authority1.prefixes.every(x => authority2Prefixes.has(x.originalPrefix));
+            if (isSubset) {
+                return doRangesOverlap;
             }
         }
-        // If any of their prefixes are under the prefix and match it, then that's a match. 
-        if (authority1.prefixes.some(x => this.matchesAuthoritySpec(authority2, x))) {
-            return true;
-        }
-        if (authority2.prefixes.some(x => this.matchesAuthoritySpec(authority1, x))) {
-            return true;
-        }
-        if (authority1.excludeDefault && authority2.excludeDefault) {
-            // No shared prefixes, and none of them are nested under each other, and we don't include defaults, so neither match. 
-            return false;
+        if (authority2.excludeDefault) {
+            let authority1Prefixes = new Set(authority1.prefixes.map(x => x.originalPrefix));
+            let isSubset = authority2.prefixes.every(x => authority1Prefixes.has(x.originalPrefix));
+            if (isSubset) {
+                return doRangesOverlap;
+            }
         }
-        // If their prefixes are entirely unrelated, it means they're going to hash differently, so they do overlap. 
+
+        // Otherwise, they hash differently, so they likely have overlap. 
         return true;
     }
     @measureFnc
@@ -384,10 +435,10 @@ export class PathRouter {
             if (target.excludeDefault) {
                 // This relaxes the restrictions, as with no default hashes, it means if a value isn't in one of our prefixes, even if it's inconsistent in the sources, it's fine. 
                 // (And... Otherwise, we can't filter these at all because otherwise the default/remaining group will be different, even if it's a superset/subset relationship)
-                let targetUsedPrefixes = new Set(target.prefixes);
-                usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x));
+                let targetUsedPrefixes = new Set(target.prefixes.map(x => x.originalPrefix));
+                usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x.originalPrefix));
             }
-            let prefixHash = sha256(JSON.stringify(usedPrefixes.sort()));
+            let prefixHash = sha256(JSON.stringify(usedPrefixes.map(x => x.originalPrefix).sort()));
             let group = groupByPrefixHash.get(prefixHash);
             if (!group) {
                 group = [];
@@ -403,17 +454,17 @@ export class PathRouter {
         }
 
         function hashesSameAsTarget(spec: AuthoritySpec): boolean {
-            let targetUsedPrefixes = new Set(target.prefixes);
+            let targetUsedPrefixes = new Set(target.prefixes.map(x => x.originalPrefix));
             if (target.excludeDefault) {
                 // If target is excluding default, then it doesn't matter if the spec is including it. We're only looking for matching values in target.
-                return spec.prefixes.every(x => targetUsedPrefixes.has(x));
+                return spec.prefixes.every(x => targetUsedPrefixes.has(x.originalPrefix));
             }
             // if !targetExcludeDefault, then !spec.excludeDefault, because we filtered out other cases earlier (so we don't need to handle spec.
 
             if (spec.prefixes.length !== target.prefixes.length) return false;
             // Otherwise, All of the prefixes have to be identical
             for (let prefix of spec.prefixes) {
-                if (!targetUsedPrefixes.has(prefix)) return false;
+                if (!targetUsedPrefixes.has(prefix.originalPrefix)) return false;
             }
             return true;
         }
@@ -498,7 +549,7 @@ export class PathRouter {
 
 
         // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
-        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y === path)).map(x => x.authoritySpec);
+        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y.prefix === path)).map(x => x.authoritySpec);
         if (hasPrefix.length > 0) {
             shuffle(hasPrefix, Math.random());
             sort(hasPrefix, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
@@ -534,11 +585,11 @@ export class PathRouter {
         let nestedMatches = allSources.filter(x => {
             // There's nested prefixes, so if we match any prefix explicitly, we can't just take one of the previous prefixes because that isn't how the hashing will work. 
             // - This happens if it's a direct match, but one of the shards is down, in which case we can't get a full match.
-            if (x.authoritySpec.prefixes.some(y => y === path)) return false;
+            if (x.authoritySpec.prefixes.some(y => y.prefix === path)) return false;
 
             // If our path, which we're going to read the children of, is the child of another path, then it means in that other path, the child key will be known to us constant, and so we're going to match exactly one authority. 
             return (
-                x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
+                x.authoritySpec.prefixes.some(y => matchesPrefix(y, path))
                 && this.matchesAuthoritySpec(x.authoritySpec, path)
             );
         });
@@ -558,7 +609,7 @@ export class PathRouter {
 
         // If we are not under any prefixes of it, then it will be a full path hash
         let fullPathMatches = allSources.filter(x => {
-            return !x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path) && !x.authoritySpec.excludeDefault;
+            return !x.authoritySpec.prefixes.some(y => matchesPrefix(y, path)) && !x.authoritySpec.excludeDefault;
         });
         // Same as prefix matches. Not preferred, and not preferred over being under a prefix, but required for some root data, or data with no prefixes.
         if (fullPathMatches.length > 0) {

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -62,7 +62,7 @@ export const hasPrefixHash = cacheLimited(1000 * 10,
     (config: { spec: AuthoritySpec, prefixHash: string }) => {
         let { spec, prefixHash } = config;
         for (let prefix of spec.prefixes) {
-            if (getPrefixHash(prefix) === prefixHash) {
+            if (getPrefixHash(prefix.originalPrefix) === prefixHash) {
                 return true;
             }
         }

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -1,5 +1,5 @@
 import path from "path";
-import { AuthoritySpec } from "./PathRouter";
+import { AuthoritySpec, parsePrefixMatcher } from "./PathRouter";
 import { getPathDepth } from "../path";
 import { getAuthorityRange, getAuthorityExcludeDefault, getAuthorityPrefix } from "../config";
 import { getShardPrefixes } from "./ShardPrefixes";
@@ -9,6 +9,7 @@ export async function getOurAuthoritySpec(defaultToAll: true): Promise<Authority
 export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<AuthoritySpec | undefined>;
 export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<AuthoritySpec | undefined> {
     let prefixes = await getShardPrefixes();
+    let prefixMatchers = prefixes.map(p => parsePrefixMatcher(p));
 
     const range = getAuthorityRange();
     const excludeDefault = getAuthorityExcludeDefault();
@@ -20,7 +21,7 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
                 nodeId: "",
                 routeStart: 0,
                 routeEnd: 1,
-                prefixes: prefixes,
+                prefixes: prefixMatchers,
             };
         }
         return undefined;
@@ -34,9 +35,9 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
         throw new Error(`Invalid authority range, should be in the format "0.5-1.0": ${range}`);
     }
 
-    let usePrefixes = prefixes;
+    let usePrefixes = prefixMatchers;
     if (cmdPrefixes && cmdPrefixes.length > 0) {
-        usePrefixes = cmdPrefixes;
+        usePrefixes = cmdPrefixes.map(p => parsePrefixMatcher(p));
     }
 
     return {
@@ -60,10 +61,11 @@ export function getEmptyAuthoritySpec(): AuthoritySpec {
 }
 export async function getAllAuthoritySpec(): Promise<AuthoritySpec> {
     let prefixes = await getShardPrefixes();
+    let prefixMatchers = prefixes.map(p => parsePrefixMatcher(p));
     return {
         nodeId: "",
         routeStart: 0,
         routeEnd: 1,
-        prefixes: prefixes,
+        prefixes: prefixMatchers,
     };
 }

package/src/0-path-value-core/PathValueCommitter.ts

@@ -15,7 +15,7 @@ import debugbreak from "debugbreak";
 import { red } from "socket-function/src/formatting/logColors";
 import { isClient } from "../config2";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { authorityLookup } from "./AuthorityLookup";
+import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { debugNodeId } from "../-c-identity/IdentityController";
 import { decodeNodeId } from "../-a-auth/certs";
 import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
@@ -28,6 +28,10 @@ setImmediate(() => import("../4-querysub/Querysub"));
 
 const MAX_SEND_TRY_COUNT = 3;
 
+// NOTE: This isn't very efficient, but it is safer. It's quadratic depending on the number of authorities for the server. Which shouldn't be so bad. I mean, why would we have four times redundancy? That's a lot. And even then, we're sending sixteen times the amount of traffic. That's fine, I guess. I think it's more reasonable for us to have two times redundancy, maybe even three times redundancy. With two times redundancy, it's only sending the value four times, which is reasonable. And the trade-off is there's basically no way that we can lose the data.
+//      - If we only send to one authority and then they rebroadcast it, it's possible that they die after receiving the value. If we send it to both value servers but they don't re-broadcast, it's possible that we die while broadcasting the values after having only sent a few. However, by sending it to both servers and having both servers send it to the other servers, it means that if we die while broadcasting it, it's fine. And if the server dies that we want to send it to, it's fine because we send it to both servers.
+const BROADCAST_TO_ALL_AUTHORITIES = true;
+
 export type BatchValues = {
     pathValues: PathValue[],
     parentsSynced?: string[];
@@ -207,8 +211,9 @@ class PathValueCommitter {
                         source: pathValue.source,
                         otherAuthorities,
                     });
+                    continue;
                 }
-                for (let otherAuthority of otherAuthorities) {
+                function sendToAuthority(otherAuthority: AuthorityEntry) {
                     let values = valuesPerOtherAuthority.get(otherAuthority.nodeId);
                     if (!values) {
                         values = [];
@@ -216,6 +221,15 @@ class PathValueCommitter {
                     }
                     values.push(pathValue);
                 }
+                if (BROADCAST_TO_ALL_AUTHORITIES) {
+                    for (let otherAuthority of otherAuthorities) {
+                        sendToAuthority(otherAuthority);
+                    }
+                } else {
+                    // NOTE: Path routing is pretty strict. Once we disconnect, it'll remove it from the list of authorities. And if we can't talk to it, the retry logic will call us again.
+                    let otherAuthority = otherAuthorities[~~(Math.random() * otherAuthorities.length)];
+                    sendToAuthority(otherAuthority);
+                }
             }
 
             // Don't send to bad nodes for 60 seconds
@@ -268,30 +282,41 @@ class PathValueCommitter {
                         });
                     }
                 });
-                void forwardPromise.catch(async error => {
-                    let byTryCount = new Map<number, PathValue[]>();
-                    for (let value of values) {
-                        let tryCount = tryCountPerValue.get(value) ?? 0;
-                        let arr = byTryCount.get(tryCount) ?? [];
-                        arr.push(value);
-                        byTryCount.set(tryCount, arr);
-                    }
-                    for (let [tryCount, values] of byTryCount.entries()) {
-                        tryCount++;
-                        if (tryCount > MAX_SEND_TRY_COUNT) {
-                            console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
-                                error: error.message,
-                                otherAuthority,
-                                count: values.length,
+                // If we broadcast to all authorities, we can't also retry, as this means if one authority is down, we end up infinitely looping. And it breaks the servers and the logs. 
+                if (!BROADCAST_TO_ALL_AUTHORITIES) {
+                    void forwardPromise.catch(async error => {
+                        let byTryCount = new Map<number, PathValue[]>();
+                        for (let value of values) {
+                            let tryCount = tryCountPerValue.get(value) ?? 0;
+                            let arr = byTryCount.get(tryCount) ?? [];
+                            arr.push(value);
+                            byTryCount.set(tryCount, arr);
+                        }
+                        for (let [tryCount, values] of byTryCount.entries()) {
+                            tryCount++;
+                            if (tryCount > MAX_SEND_TRY_COUNT) {
+                                console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    count: values.length,
+                                });
+                                continue;
+                            }
+                            for (let value of values) {
+                                console.error(`Retrying to send values after ${tryCount} tries.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    path: value.path,
+                                    timeId: value.time.time,
+                                });
+                            }
+                            void pathValueCommitter.broadcastValues({
+                                values: new Set(values),
+                                tryCount: tryCount,
                             });
-                            continue;
                         }
-                        void pathValueCommitter.broadcastValues({
-                            values: new Set(values),
-                            tryCount: tryCount,
-                        });
-                    }
-                });
+                    });
+                }
 
                 pathValueCommitter.addCommitPromise(forwardPromise);
             });
@@ -352,9 +377,10 @@ class PathValueCommitter {
                             return true;
                         });
 
-                        for (let value of Array.from(batch.initialTriggers.values)) {
-                            if (isWrongAuthority(value, undefined, "initialTrigger")) {
-                                batch.initialTriggers.values.delete(value);
+                        // NOTE: If we are the authority on it, it means we're not going to have a remote watch node ID. And if we did, it would be ourselves. So this also implicitly filters out any initial triggers for values that we are the authority on. 
+                        for (let path of Array.from(batch.initialTriggers.values)) {
+                            if (isWrongAuthority(path, undefined, "initialTrigger")) {
+                                batch.initialTriggers.values.delete(path);
                             }
                         }
                         for (let parentPath of Array.from(batch.initialTriggers.parentPaths)) {
@@ -362,11 +388,6 @@ class PathValueCommitter {
                                 continue;
                             }
 
-                            // TODO: Remove this breakpoint eventually. This can happen naturally when servers go down?
-                            require("debugbreak")(2);
-                            debugger;
-                            remoteWatcher.isFinalRemoteWatchPath({ parentPath, nodeId: batch.sourceNodeId });
-
                             console.warn(`Ignoring parent path which we aren't watching. From ${debugNodeId(batch.sourceNodeId)}.`, {
                                 parentPath,
                                 sourceNodeId: debugNodeId(batch.sourceNodeId),

package/src/0-path-value-core/PathValueController.ts

@@ -310,8 +310,9 @@ export class PathValueControllerBase {
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
         let values: PathValue[] = [];
         for (let entry of entries) {
-            let value = authorityStorage.getValueExactIgnoreInvalid(entry.path, entry.time);
+            let value = authorityStorage.getValueExact(entry.path, entry.time);
             if (!value) continue;
+            if (!value.valid) continue;
             if (value.isTransparent) continue;
             if (value.canGCValue) continue;
             values.push(value);

package/src/0-path-value-core/ShardPrefixes.ts

@@ -14,6 +14,9 @@ import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
 import { getDomain } from "../config";
+import { sort } from "socket-function/src/misc";
+import { getPathFromStr, getPathStr } from "../path";
+import { green, red } from "socket-function/src/formatting/logColors";
 
 type PrefixObj = {
     prefixes: string[];
@@ -30,8 +33,25 @@ export async function getShardPrefixes(): Promise<string[]> {
 
 export async function setShardPrefixes(_prefixes: string[]) {
     console.log("Setting shard prefixes");
+    _prefixes = _prefixes.slice();
+
+    const oldPrefixes = await getShardPrefixes();
+    const oldSet = new Set(oldPrefixes);
+    const newSet = new Set(_prefixes);
+
     for (let prefix of _prefixes) {
-        console.log("Prefix:", prefix);
+        if (!oldSet.has(prefix)) {
+            console.log(green(`Prefix: ${prefix}`));
+        } else {
+            console.log(`Prefix: ${prefix}`);
+        }
     }
+
+    for (let prefix of oldPrefixes) {
+        if (!newSet.has(prefix)) {
+            console.log(red(`Prefix: ${prefix}`));
+        }
+    }
+
     await prefixes.set(key, { prefixes: _prefixes });
 }

package/src/0-path-value-core/ValidStateComputer.ts

@@ -2,7 +2,7 @@ import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { isDiskAudit } from "../config";
-import { getPathFromStr } from "../path";
+import { getParentPathStr, getPathFromStr } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
 import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE } from "./pathValueCore";
@@ -40,13 +40,36 @@ class ValidStateComputer {
 
         let ourValues: PathValue[] = [];
         let notOurValues: PathValue[] = [];
-        for (let value of pathValues) {
+        pathValues = pathValues.filter(value => {
+            // NOTE: I think if it's initial trigger we need to process it anyways?
+            if (!config.initialTriggers.values.has(value.path)) {
+                // Ignore values we already have. We receive duplicates, often due to values being broadcast multiple times in order to prevent data loss.
+                let existingValue = authorityStorage.getValueExact(value.path, value.time);
+                // If we already have the value and the valid state is the exact same, then remove it from path values and don't even add it to our values. Pretend like we never received it.
+                if (existingValue && !!existingValue.valid === !!value.valid) {
+                    let parentPath = getParentPathStr(value.path);
+                    // We check for the parent trigger late because getting the parent path is a little bit expensive
+                    if (!config.initialTriggers.parentPaths.has(parentPath)) {
+                        console.info(`Ignoring duplicate value.`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            timeIdVersion: value.time.version,
+                            timeIdFull: value.time,
+                            valid: value.valid,
+                        });
+                        // NOTE: This should be safe. If this causes any problems, it will probably be if we're not the authority on it. In which case, we could move this check so it only happens if we are the authority on it. However, I think even if we aren't the authority, if we already have the exact value and the valid state hasn't changed, we really shouldn't have to ingest it. Watchers should be triggering when they start watching, so they shouldn't need us to trigger them again by re-ingesting it. 
+                        return false;
+                    }
+                }
+            }
+
             if (!PathRouter.isSelfAuthority(value.path)) {
                 notOurValues.push(value);
             } else {
                 ourValues.push(value);
             }
-        }
+            return true;
+        });
         authorityStorage.ingestValues(notOurValues, { doNotArchive });
 
 
@@ -93,7 +116,6 @@ class ValidStateComputer {
             }
         }
 
-
         let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
         authorityStorage.ingestValues(ourValues, { doNotArchive });
 

package/src/0-path-value-core/pathValueCore.ts

@@ -431,7 +431,7 @@ class AuthorityPathValueStorage {
     public getValue(path: string): unknown {
         return pathValueSerializer.getPathValue(this.getValueAtTime(path));
     }
-    public getValueExactIgnoreInvalid(path: string, time: Time): PathValue | undefined {
+    public getValueExact(path: string, time: Time): PathValue | undefined {
         let value = this.getValueAtTime(path, time, true);
         if (value && compareTime(value.time, time) === 0) {
             return value;

package/src/2-proxy/schema2.ts

@@ -150,6 +150,19 @@ type TypeDefType<T = never, Optional = false> = {
         schema: NewT
     ): TypeDefType<{ [key: string | number]: TypeDefToT<NewT> }>;
 
+    /** Lookup with GC enabled, but excluded from root lookup routing.
+     *      Behaves like a regular lookup but won't be used as a prefix for sharding.
+     */
+    lookupNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: string]: TypeDefToT<NewT> }>;
+    lookupNumberNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: number]: TypeDefToT<NewT> }>;
+    lookupEitherNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: string | number]: TypeDefToT<NewT> }>;
+
 
 
     // TODO: Support .union, which just spreads the inputs together, making an object which
@@ -247,22 +260,42 @@ export class Schema2Fncs {
         return schema2ToTypeString(schema);
     }
 
-    /** Uses * for wildcards */
-    public static getLookups(schema: Schema2): string[][] {
-        let internalType = typeDefCached(schema);
-        let lookupPaths = new Set<string>();
-
-        for (let def of internalType.defs) {
-            for (let i = 0; i < def.path.length; i++) {
-                if (def.path[i] === WildCard) {
-                    let lookupPath = def.path.slice(0, i);
-                    let stringPath = lookupPath.map(p => p === WildCard ? "*" : p as string);
-                    lookupPaths.add(JSON.stringify(stringPath));
+    public static getRootLookups(schema: Schema2): string[][] {
+        let defs = typeDefCached(schema).defs.slice();
+
+        sort(defs, x => x.path.length);
+
+        let rootLookups = new Set<string>();
+        for (let def of defs) {
+            if (def.isLookupNoRoot) continue;
+            if (!def.isObjectGC) continue;
+            let path = def.path;
+            if (path.at(-1) !== WildCard) continue;
+            path = path.slice(0, -1);
+
+            let pathParts = path.map(p => p === WildCard ? "" : p);
+            let pathKey = JSON.stringify(pathParts);
+
+            let hasAncestor = false;
+            for (let i = 0; i < path.length - 1; i++) {
+                let ancestorPath = path.slice(0, i + 1);
+                let ancestorParts = ancestorPath.map(p => p === WildCard ? "" : p);
+                let ancestorKey = JSON.stringify(ancestorParts);
+                if (rootLookups.has(ancestorKey)) {
+                    hasAncestor = true;
+                    break;
                 }
             }
+
+            if (!hasAncestor) {
+                rootLookups.add(pathKey);
+            }
         }
 
-        return Array.from(lookupPaths).map(p => JSON.parse(p));
+        return Array.from(rootLookups).map(p => {
+            let parts = JSON.parse(p) as string[];
+            return parts;
+        });
     }
 }
 
@@ -518,6 +551,8 @@ type InternalTypeDef = {
     isBoolean?: boolean;
     isNull?: boolean;
 
+    isLookupNoRoot?: boolean;
+
     gcDelay?: number;
 
     defaultValue?: unknown;
@@ -544,6 +579,7 @@ function typeDefTypeToInternalType(
     }
     let isLookup = false;
     let isLookupGC = false;
+    let isLookupNoRoot = false;
     if (typeDef.path.includes("lookup") || typeDef.path.includes("lookupNumber") || typeDef.path.includes("lookupEither")) {
         isLookup = true;
         isLookupGC = true;
@@ -552,6 +588,11 @@ function typeDefTypeToInternalType(
         isLookup = true;
         isLookupGC = false;
     }
+    if (typeDef.path.includes("lookupNoRoot") || typeDef.path.includes("lookupNumberNoRoot") || typeDef.path.includes("lookupEitherNoRoot")) {
+        isLookup = true;
+        isLookupGC = true;
+        isLookupNoRoot = true;
+    }
 
     let isObject = false;
     let isObjectGC = false;
@@ -594,6 +635,7 @@ function typeDefTypeToInternalType(
         // NOTE: Gets reset if the nestedValues are used with a primitive
         nestedValues.isNotAtomic = true;
         nestedValues.isObjectGC = isObjectGC || isLookupGC;
+        nestedValues.isLookupNoRoot = isLookupNoRoot;
         nestedValues.gcDelay = Number(typeDef.pathWithCalls.find(x => x.key === "gcDelay")?.callParams?.[0]) || undefined;
 
 

package/src/3-path-functions/syncSchema.ts

@@ -372,27 +372,11 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
                 };
             }
 
-            // Register all root lookups as prefixes.
             let dataRoot = getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Data);
-            let paths = Schema2Fncs.getLookups(schema).map(x =>
-                joinPathStres(dataRoot, getPathStr(x))
-            );
-            // NOTE: This will break with wildcards, but... we also take only the highest level lookups, so it shouldn't be an issue
-            sort(paths, x => x.length);
-            let rootLookups = new Set<string>();
-            for (let path of paths) {
-                let parts = getPathFromStr(path);
-                // Don't add if any of our ancestors are added
-                function hasAncestor() {
-                    for (let i = 0; i < parts.length - 1; i++) {
-                        let ancestorPath = getPathStr(parts.slice(0, i));
-                        if (rootLookups.has(ancestorPath)) return true;
-                    }
-                }
-                if (!hasAncestor()) {
-                    rootLookups.add(path);
-                    prefixes.push(path);
-                }
+            let rootLookups = Schema2Fncs.getRootLookups(schema);
+            for (let lookup of rootLookups) {
+                let path = joinPathStres(dataRoot, getPathStr(lookup));
+                prefixes.push(path);
             }
         }
 

package/src/4-deploy/deployPrefixes.ts

@@ -7,6 +7,9 @@ async function main() {
     await import(deployPath);
 
     let prefixes = await getPrefixesForDeploy();
+    for (let prefix of prefixes) {
+        console.log(prefix);
+    }
     await setShardPrefixes(prefixes);
 }
 

package/src/4-querysub/Querysub.ts

@@ -1026,8 +1026,6 @@ export class Querysub {
         RequireController.addMapGetModules(async (result, args) => {
             let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
             if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
-                require("debugbreak")(2);
-                debugger;
                 await isAllowedToSeeSource(configObj?.signedIdentity);
                 //console.log(red(`Not allowed to see source`));
                 for (let [key, value] of Object.entries(result.modules)) {

package/src/diagnostics/SyncTestPage.tsx

@@ -16,6 +16,8 @@ import { sort } from "socket-function/src/misc";
 import { PathAuditerController } from "./pathAuditer";
 import { t } from "../2-proxy/schema2";
 
+//todonext
+// So... We need to double write, to write to multiple keys at once, in something that's not a lookup, or no, is a lookup, but before we deploy, so it doesn't have the proper or no, a nested lookup, even better. 
 
 let { data, functions } = Querysub.createSchema({
     values: t.lookup(t.number)

package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts

@@ -13,7 +13,8 @@ export const STREAMING_BLOCK_THRESHOLD = 1024 * 1024;
 
 export const PUBLIC_MOVE_THRESHOLD = timeInHour * 3;
 // Max uncompressed data size
-export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 512;
+// NOTE: This used to be 512MB, but searching was really slow. I think making this smaller will make searching faster?
+export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 64;
 
 export const MOVING_TIMEOUT = timeInMinute * 5;
 

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -479,8 +479,7 @@ export class IndexedLogs<T> {
         if (onResultsCallback) {
             interval = setInterval(async () => {
                 updateResultsStats();
-                let results = getFinalResults();
-                let shouldContinue = await onResultsCallback(results);
+                let shouldContinue = await onResultsCallback(getFinalResults());
                 if (!shouldContinue) {
                     if (!results.cancel) {
                         console.log(blue(`Cancelled search on ${this.config.name}`));

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -6,7 +6,7 @@ import { BufferIndex } from "./BufferIndex";
 import { LogStreamer } from "./LogStreamer";
 import { TimeFileTree } from "./TimeFileTree";
 import { blue, green, magenta } from "socket-function/src/formatting/logColors";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 
 
 
@@ -150,15 +150,15 @@ export async function moveLogsToPublic(config: {
     let groups: typeof initialGroups = [];
     for (let group of initialGroups) {
         let currentSize = 0;
-        
+
         for (let path of group) {
             let size = sizeMap.get(path) || 0;
-            
+
             if (groups.length === 0 || currentSize + size > sizeThreshold) {
                 groups.push([]);
                 currentSize = 0;
             }
-            
+
             groups[groups.length - 1].push(path);
             currentSize += size;
         }
@@ -256,7 +256,7 @@ export async function moveLogsToPublic(config: {
             await localLogs.del(path.fullPath + indexExtension);
         }));
 
-        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
+        console.log(green(`(${i + 1}/${groups.length}) Wrote ${encoded.length} log files to public from ${group.length} input files, ${formatDateTime(startTime)} to ${formatDateTime(endTime)} (${formatNumber(encoded.reduce((acc, x) => acc + x.uncompressedSize, 0))}B compressed to ${formatNumber(encoded.reduce((acc, x) => acc + x.compressedSize, 0))}B + ${formatNumber(encoded.reduce((acc, x) => acc + x.index.length, 0))}B index) in ${formatTime(Date.now() - time)}`) + ` (${blue(config.loggerName)})`);
     }
 
     // Clean up orphaned index files (index files without corresponding data files)

package/src/diagnostics/logs/ObjectDisplay.tsx

@@ -99,7 +99,7 @@ export class ObjectDisplay extends qreact.Component<{
                 <span className={css.maxHeight("70vh").overflowAuto} onClick={log}>
                     <div class={css.relative}>
                         {renderValue(value, [])}
-                        <MeasureHeightCSS />
+                        {/* <MeasureHeightCSS /> */}
                     </div>
                 </span>
             </ShowMore>;

package/test.ts

@@ -16,6 +16,7 @@ import { ClientWatcher } from "./src/1-path-client/pathValueClientWatcher";
 import { RemoteWatcher } from "./src/1-path-client/RemoteWatcher";
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { shutdown } from "./src/diagnostics/periodic";
+import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
 
 let tempTestSchema = Querysub.createSchema({
     value: t.number,
@@ -27,7 +28,12 @@ let tempTestSchema = Querysub.createSchema({
 });
 
 async function main() {
-    await Querysub.hostService("test");
+
+    let prefixes = await getShardPrefixes();
+    for (let prefix of prefixes) {
+        console.log(prefix);
+    }
+    // await Querysub.hostService("test");
 
     // let path = getProxyPath(() => tempTestSchema.data().value);
     // console.log({ path });
@@ -40,14 +46,14 @@ async function main() {
     // ClientWatcher.DEBUG_WRITES = true;
     // RemoteWatcher.DEBUG = true;
 
-    let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
-    console.log({ value });
-    await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
-    await delay(3000);
-    let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
-    console.log({ value2 });
+    // let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    // console.log({ value });
+    // await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
+    // await delay(3000);
+    // let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    // console.log({ value2 });
 
-    await shutdown();
+    // await shutdown();
 
     // let test = await Querysub.commitAsync(() => {
     //     let live = deploySchema()[getDomain()].deploy.live.hash;