querysub 0.425.0 → 0.427.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.425.0",
+  "version": "0.427.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/AuthorityLookup.ts

@@ -12,7 +12,7 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isClient } from "../config2";
 import { getPathStr1 } from "../path";
 import { timeoutToError } from "../errors";
-import { AuthoritySpec } from "./PathRouter";
+import { AuthoritySpec, parsePrefixMatcher } from "./PathRouter";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getAllAuthoritySpec, getEmptyAuthoritySpec } from "./PathRouterServerAuthoritySpec";
 
@@ -208,10 +208,10 @@ class AuthorityLookup {
             //  - Get all node IDs should restrict our nodes to just the browser node ID. If we ever change this, then either it's redundant nodes and they all have all the same data, or we need to figure out what data they have, And as their proxies, it probably won't be their actual authority data. So that will require new API functions, etc. 
             await new Promise(r => setImmediate(r));
             await delay(1);
-            let { getPrefixesForDeploy } = await import("../3-path-functions/syncSchema");
+            let prefixes = await this.getPrefixesForDeploy();
             this.updatePaths(nodeId, {
                 nodeId: nodeId,
-                prefixes: await getPrefixesForDeploy(),
+                prefixes: prefixes.map(p => parsePrefixMatcher(p)),
                 routeStart: 0,
                 routeEnd: 1,
             }, true);
@@ -233,6 +233,11 @@ class AuthorityLookup {
         this.updatePaths(nodeId, otherPaths.spec, otherPaths.isReady);
         this.disconnectWatch(nodeId);
     }
+
+    private async getPrefixesForDeploy() {
+        let { getPrefixesForDeploy } = await import("../3-path-functions/syncSchema");
+        return await getPrefixesForDeploy();
+    }
 }
 export const authorityLookup = new AuthorityLookup();
 (globalThis as any).authorityLookup = authorityLookup;

package/src/0-path-value-core/PathRouter.ts

@@ -1,5 +1,5 @@
 import { sort } from "socket-function/src/misc";
-import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1, hack_stripPackedPath } from "../path";
+import { getLastPathPart, getPathDepth, getPathFromStr, getPathIndex, getPathStr, getPathStr1, hack_stripPackedPath } from "../path";
 import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
@@ -33,10 +33,65 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
-    prefixes: string[];
+    prefixes: PrefixMatcher[];
     excludeDefault?: boolean;
 };
 
+
+export type PrefixMatcher = {
+    prefix: string;
+    additionalMatches: {
+        value: string;
+        index: number;
+    }[];
+    childKeyIndex: number;
+    originalPrefix: string;
+    prefixPartLength: number;
+};
+function matchesPrefix(matcher: PrefixMatcher, path: string): boolean {
+    if (!path.startsWith(matcher.prefix)) return false;
+    if (matcher.prefix === path) return false;
+    if (getPathIndex(path, matcher.childKeyIndex) === undefined) return false;
+    for (let additionalMatch of matcher.additionalMatches) {
+        if (getPathIndex(path, additionalMatch.index) !== additionalMatch.value) {
+            return false;
+        }
+    }
+    return true;
+}
+/** Empty string path part becomes a wildcard */
+export function parsePrefixMatcher(prefixPath: string): PrefixMatcher {
+    // So up until the first wild card, that's a prefix. But then after that, we then need to skip all the wild cards, of course. And then for the non-wild cards, add them as additional matches. And then obviously set the child key index to just be the part's length. 
+    let parts = getPathFromStr(prefixPath);
+    let wildcardIndex = parts.indexOf("");
+    if (wildcardIndex === -1) {
+        return {
+            prefix: prefixPath,
+            additionalMatches: [],
+            childKeyIndex: parts.length,
+            originalPrefix: prefixPath,
+            prefixPartLength: parts.length,
+        };
+    }
+    let prefix = getPathStr(parts.slice(0, wildcardIndex));
+    let additionalMatches: { value: string; index: number }[] = [];
+    for (let i = wildcardIndex + 1; i < parts.length; i++) {
+        let part = parts[i];
+        if (part === "") continue;
+        additionalMatches.push({
+            value: part,
+            index: i,
+        });
+    }
+    return {
+        prefix,
+        additionalMatches,
+        childKeyIndex: wildcardIndex,
+        originalPrefix: prefixPath,
+        prefixPartLength: parts.length,
+    };
+}
+
 export function debugSpec(spec: AuthoritySpec) {
     return {
         info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,
@@ -44,16 +99,16 @@ export function debugSpec(spec: AuthoritySpec) {
     };
 }
 
-function getMatchingPrefix(spec: AuthoritySpec, path: string): string | undefined {
-    let longestPrefix: string | undefined;
-    for (let prefix of spec.prefixes) {
-        if (path.startsWith(prefix) && prefix !== path) {
-            if (!longestPrefix || longestPrefix.length < prefix.length) {
-                longestPrefix = prefix;
+function getMatchingPrefix(spec: AuthoritySpec, path: string): PrefixMatcher | undefined {
+    let longestMatcher: PrefixMatcher | undefined;
+    for (let matcher of spec.prefixes) {
+        if (matchesPrefix(matcher, path)) {
+            if (!longestMatcher || longestMatcher.prefixPartLength < matcher.prefixPartLength) {
+                longestMatcher = matcher;
             }
         }
     }
-    return longestPrefix;
+    return longestMatcher;
 }
 
 
@@ -81,11 +136,11 @@ export class PathRouter {
 
         let prefix = getMatchingPrefix(spec, path);
         if (prefix) {
-            let key = getPathIndex(path, getPathDepth(prefix));
+            let key = getPathIndex(path, prefix.childKeyIndex);
             if (key === undefined) {
                 require("debugbreak")(2);
                 debugger;
-                throw new Error(`Impossible, hash index ${getPathDepth(prefix)} is out of range for path ${path}, but it matched the prefix ${prefix}`);
+                throw new Error(`Impossible, hash index ${prefix.childKeyIndex} is out of range for path ${path}, but it matched the prefix ${prefix.originalPrefix}`);
             }
             let route = this.getSingleKeyRoute(key);
             if (route < spec.routeStart || route >= spec.routeEnd) return -1;
@@ -136,13 +191,13 @@ export class PathRouter {
     }
 
 
-    private static getPrefixHash(prefix: string): string {
-        return Buffer.from(sha256(prefix), "hex").toString("base64").slice(0, 6);
+    private static getPrefixHash(prefix: PrefixMatcher): string {
+        return Buffer.from(sha256(prefix.originalPrefix), "hex").toString("base64").slice(0, 6);
     }
     private static isPrefixHash(hash: string): boolean {
         return hash.length === 6 && /^[a-zA-Z0-9]+$/.test(hash);
     }
-    private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
+    private static encodeIdentifier(config: { prefixes: PrefixMatcher[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
         return [
@@ -151,7 +206,7 @@ export class PathRouter {
             rangeEnd.toString(),
             ...prefixes.map(x => this.getPrefixHash(x)),
             // Add some prefixes. They'll be interpreted as hashes, but it's fine because it's highly unlikely they'll collide with a real hash. AND... It's very useful for debugging what's in which file. 
-            ...prefixes.slice(0, 5).map(x => getLastPathPart(x).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
+            ...prefixes.slice(0, 5).map(x => getLastPathPart(x.originalPrefix).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
             ""
         ].join("!");
 
@@ -182,10 +237,10 @@ export class PathRouter {
         const SHARD_THRESHOLD = 1000;
 
         let prefixes = ourSpec.prefixes.slice();
-        sort(prefixes, x => x.length);
+        sort(prefixes, x => x.prefixPartLength);
 
         // NOTE: If there are few enough path values for a prefix, we don't even need to calculate the routing hash. 
-        let byPrefix = new Map<string | undefined, PathValue[]>();
+        let byPrefix = new Map<PrefixMatcher | undefined, PathValue[]>();
         for (let value of values) {
             let prefix = getMatchingPrefix(ourSpec, value.path);
             let values = byPrefix.get(prefix);
@@ -203,7 +258,7 @@ export class PathRouter {
         sort(prefixGroups, x => -x.values.length);
 
         let groups: {
-            prefixes: string[];
+            prefixes: PrefixMatcher[];
             values: PathValue[][];
             count: number;
         }[] = [];
@@ -305,34 +360,30 @@ export class PathRouter {
     public static overlapsAuthority(authority1: AuthoritySpec, authority2: AuthoritySpec): boolean {
         // TODO: This becomes complicated because of exclude default, although I feel like there has to be a way to simplify it? Eh... whatever.
 
-        // Normalize it so if only one excludes default, it's always going to be the second one. 
-        if (authority1.excludeDefault && !authority2.excludeDefault) return this.overlapsAuthority(authority2, authority1);
-
         let doRangesOverlap = rangesOverlap({ start: authority1.routeStart, end: authority1.routeEnd }, { start: authority2.routeStart, end: authority2.routeEnd });
 
         // If their prefixes are identical, then it's purely a range check
-        if (authority1.prefixes.length === authority2.prefixes.length && authority1.prefixes.every(x => authority2.prefixes.includes(x))) {
+        if (authority1.prefixes.length === authority2.prefixes.length && authority1.prefixes.every(x => authority2.prefixes.some(y => x.originalPrefix === y.originalPrefix))) {
             return doRangesOverlap;
         }
 
-        // If they have any prefixes which are identical and the ranges overlap, then they overlap.
-        if (doRangesOverlap) {
-            if (authority1.prefixes.some(x => authority2.prefixes.includes(x))) {
-                return true;
+        // If one has excludeDefault and is a subset of the other's prefixes, then it's a range check
+        if (authority1.excludeDefault) {
+            let authority2Prefixes = new Set(authority2.prefixes.map(x => x.originalPrefix));
+            let isSubset = authority1.prefixes.every(x => authority2Prefixes.has(x.originalPrefix));
+            if (isSubset) {
+                return doRangesOverlap;
             }
         }
-        // If any of their prefixes are under the prefix and match it, then that's a match. 
-        if (authority1.prefixes.some(x => this.matchesAuthoritySpec(authority2, x))) {
-            return true;
-        }
-        if (authority2.prefixes.some(x => this.matchesAuthoritySpec(authority1, x))) {
-            return true;
-        }
-        if (authority1.excludeDefault && authority2.excludeDefault) {
-            // No shared prefixes, and none of them are nested under each other, and we don't include defaults, so neither match. 
-            return false;
+        if (authority2.excludeDefault) {
+            let authority1Prefixes = new Set(authority1.prefixes.map(x => x.originalPrefix));
+            let isSubset = authority2.prefixes.every(x => authority1Prefixes.has(x.originalPrefix));
+            if (isSubset) {
+                return doRangesOverlap;
+            }
         }
-        // If their prefixes are entirely unrelated, it means they're going to hash differently, so they do overlap. 
+
+        // Otherwise, they hash differently, so they likely have overlap. 
         return true;
     }
     @measureFnc
@@ -384,10 +435,10 @@ export class PathRouter {
             if (target.excludeDefault) {
                 // This relaxes the restrictions, as with no default hashes, it means if a value isn't in one of our prefixes, even if it's inconsistent in the sources, it's fine. 
                 // (And... Otherwise, we can't filter these at all because otherwise the default/remaining group will be different, even if it's a superset/subset relationship)
-                let targetUsedPrefixes = new Set(target.prefixes);
-                usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x));
+                let targetUsedPrefixes = new Set(target.prefixes.map(x => x.originalPrefix));
+                usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x.originalPrefix));
             }
-            let prefixHash = sha256(JSON.stringify(usedPrefixes.sort()));
+            let prefixHash = sha256(JSON.stringify(usedPrefixes.map(x => x.originalPrefix).sort()));
             let group = groupByPrefixHash.get(prefixHash);
             if (!group) {
                 group = [];
@@ -403,17 +454,17 @@ export class PathRouter {
         }
 
         function hashesSameAsTarget(spec: AuthoritySpec): boolean {
-            let targetUsedPrefixes = new Set(target.prefixes);
+            let targetUsedPrefixes = new Set(target.prefixes.map(x => x.originalPrefix));
             if (target.excludeDefault) {
                 // If target is excluding default, then it doesn't matter if the spec is including it. We're only looking for matching values in target.
-                return spec.prefixes.every(x => targetUsedPrefixes.has(x));
+                return spec.prefixes.every(x => targetUsedPrefixes.has(x.originalPrefix));
             }
             // if !targetExcludeDefault, then !spec.excludeDefault, because we filtered out other cases earlier (so we don't need to handle spec.
 
             if (spec.prefixes.length !== target.prefixes.length) return false;
             // Otherwise, All of the prefixes have to be identical
             for (let prefix of spec.prefixes) {
-                if (!targetUsedPrefixes.has(prefix)) return false;
+                if (!targetUsedPrefixes.has(prefix.originalPrefix)) return false;
             }
             return true;
         }
@@ -498,7 +549,7 @@ export class PathRouter {
 
 
         // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
-        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y === path)).map(x => x.authoritySpec);
+        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y.prefix === path)).map(x => x.authoritySpec);
         if (hasPrefix.length > 0) {
             shuffle(hasPrefix, Math.random());
             sort(hasPrefix, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
@@ -534,11 +585,11 @@ export class PathRouter {
         let nestedMatches = allSources.filter(x => {
             // There's nested prefixes, so if we match any prefix explicitly, we can't just take one of the previous prefixes because that isn't how the hashing will work. 
             // - This happens if it's a direct match, but one of the shards is down, in which case we can't get a full match.
-            if (x.authoritySpec.prefixes.some(y => y === path)) return false;
+            if (x.authoritySpec.prefixes.some(y => y.prefix === path)) return false;
 
             // If our path, which we're going to read the children of, is the child of another path, then it means in that other path, the child key will be known to us constant, and so we're going to match exactly one authority. 
             return (
-                x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
+                x.authoritySpec.prefixes.some(y => matchesPrefix(y, path))
                 && this.matchesAuthoritySpec(x.authoritySpec, path)
             );
         });
@@ -558,7 +609,7 @@ export class PathRouter {
 
         // If we are not under any prefixes of it, then it will be a full path hash
         let fullPathMatches = allSources.filter(x => {
-            return !x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path) && !x.authoritySpec.excludeDefault;
+            return !x.authoritySpec.prefixes.some(y => matchesPrefix(y, path)) && !x.authoritySpec.excludeDefault;
         });
         // Same as prefix matches. Not preferred, and not preferred over being under a prefix, but required for some root data, or data with no prefixes.
         if (fullPathMatches.length > 0) {
@@ -600,7 +651,7 @@ export class PathRouter {
             // TODO: We could maybe match a partial match. However, even that is suspect. The site being partially broken is almost worse than it being completely broken. We should just get ALL the shards running again...
 
             // NOTE: We *could* actually synchronize it even if it doesn't have a prefix shard as we can fall back to just the full path sharding. However, it becomes very complicated if we want a specific range, and then it becomes complicated if it then switches to prefix hashing (With the nodes that were using the full path hashing slowly going away). AND... key synchronization IS slow, so it's good to discourage it in general. 
-            console.error(`Want to sync a prefix which is not under an existing prefix, nor equal to a prefix. 1) The servers are down. 2) Don't access the .keys() 3) call addRoutingPrefixForDeploy to add a route/parent route explicitly (as is done in PathFunctionRunner.ts). Path: ${JSON.stringify(path)}`, { path, allSources });
+            console.error(`Want to sync a prefix which is not under an existing prefix, nor equal to a prefix. 1) The servers are down. 2) Don't access the .keys() 3) call addRoutingPrefixForDeploy to add a route/parent route explicitly (as is done in PathFunctionRunner.ts). Path: ${JSON.stringify(path)}`, { path, allSources: allSources.map(x => x.authoritySpec) });
         }
         return { nodes: [] };
     }

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -62,7 +62,7 @@ export const hasPrefixHash = cacheLimited(1000 * 10,
     (config: { spec: AuthoritySpec, prefixHash: string }) => {
         let { spec, prefixHash } = config;
         for (let prefix of spec.prefixes) {
-            if (getPrefixHash(prefix) === prefixHash) {
+            if (getPrefixHash(prefix.originalPrefix) === prefixHash) {
                 return true;
             }
         }

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -1,5 +1,5 @@
 import path from "path";
-import { AuthoritySpec } from "./PathRouter";
+import { AuthoritySpec, parsePrefixMatcher } from "./PathRouter";
 import { getPathDepth } from "../path";
 import { getAuthorityRange, getAuthorityExcludeDefault, getAuthorityPrefix } from "../config";
 import { getShardPrefixes } from "./ShardPrefixes";
@@ -9,6 +9,7 @@ export async function getOurAuthoritySpec(defaultToAll: true): Promise<Authority
 export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<AuthoritySpec | undefined>;
 export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<AuthoritySpec | undefined> {
     let prefixes = await getShardPrefixes();
+    let prefixMatchers = prefixes.map(p => parsePrefixMatcher(p));
 
     const range = getAuthorityRange();
     const excludeDefault = getAuthorityExcludeDefault();
@@ -20,7 +21,7 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
                 nodeId: "",
                 routeStart: 0,
                 routeEnd: 1,
-                prefixes: prefixes,
+                prefixes: prefixMatchers,
             };
         }
         return undefined;
@@ -34,9 +35,9 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
         throw new Error(`Invalid authority range, should be in the format "0.5-1.0": ${range}`);
     }
 
-    let usePrefixes = prefixes;
+    let usePrefixes = prefixMatchers;
     if (cmdPrefixes && cmdPrefixes.length > 0) {
-        usePrefixes = cmdPrefixes;
+        usePrefixes = cmdPrefixes.map(p => parsePrefixMatcher(p));
     }
 
     return {
@@ -60,10 +61,11 @@ export function getEmptyAuthoritySpec(): AuthoritySpec {
 }
 export async function getAllAuthoritySpec(): Promise<AuthoritySpec> {
     let prefixes = await getShardPrefixes();
+    let prefixMatchers = prefixes.map(p => parsePrefixMatcher(p));
     return {
         nodeId: "",
         routeStart: 0,
         routeEnd: 1,
-        prefixes: prefixes,
+        prefixes: prefixMatchers,
     };
 }

package/src/0-path-value-core/PathWatcher.ts

@@ -369,8 +369,10 @@ class PathWatcher {
 
         // initialTriggers.parentPaths => triggerPaths
         for (let parentPath of initialTriggers.parentPaths) {
-            let valuePaths = authorityStorage.getPathsFromParent(parentPath);
+            // NOTE: We have to trigger all of the values without the hacked path restriction. This should be safe, if a little bit inefficient, because we're still not going to send the values to watchers who aren't watching them. But we need to trigger all of it because it might be the case that we had synchronized 99% of the path and someone was watching 100% of the path and we just finished synchronizing the last 1%. So the parent path trigger that we received only has 1%, but we need to send everything. And it's too complicated to determine exactly what we need to send as in the intersection of all the watchers. So it's easier to just trigger all the value paths and what they're watching will naturally be sent. 
+            let valuePaths = authorityStorage.getPathsFromParent(hack_stripPackedPath(parentPath));
             for (let valuePath of valuePaths || []) {
+                if (!authorityStorage.isSynced(valuePath)) continue;
                 // IMPORTANT! Add all the values to the initial triggers so then later we not only trigger them but know their initial trigger so we can make sure that the is initial trigger logic for each value runs as well. 
                 initialTriggers.values.add(valuePath);
                 triggerPaths.add(valuePath);
@@ -378,7 +380,9 @@ class PathWatcher {
 
             let latestParentWatches = this.parentWatchers.get(hack_stripPackedPath(parentPath));
             if (!latestParentWatches) continue;
-            for (let { watchers } of latestParentWatches.values()) {
+            for (let [packedPath, { watchers }] of latestParentWatches) {
+                // ONLY trigger, if packedPath is fully synchronized
+                if (!authorityStorage.isParentSynced(packedPath)) continue;
                 for (let watcher of watchers) {
                     if (onlyTriggerNodeId && watcher !== onlyTriggerNodeId) continue;
                     let changes = changedPerCallbacks.get(watcher);
@@ -386,7 +390,7 @@ class PathWatcher {
                         changes = { values: new Set(), initialTriggers: { values: new Set(), parentPaths: new Set() } };
                         changedPerCallbacks.set(watcher, changes);
                     }
-                    changes.initialTriggers.parentPaths.add(parentPath);
+                    changes.initialTriggers.parentPaths.add(packedPath);
                 }
             }
         }

package/src/0-path-value-core/ShardPrefixes.ts

@@ -14,6 +14,9 @@ import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
 import { getDomain } from "../config";
+import { sort } from "socket-function/src/misc";
+import { getPathFromStr, getPathStr } from "../path";
+import { green, red } from "socket-function/src/formatting/logColors";
 
 type PrefixObj = {
     prefixes: string[];
@@ -30,8 +33,25 @@ export async function getShardPrefixes(): Promise<string[]> {
 
 export async function setShardPrefixes(_prefixes: string[]) {
     console.log("Setting shard prefixes");
+    _prefixes = _prefixes.slice();
+
+    const oldPrefixes = await getShardPrefixes();
+    const oldSet = new Set(oldPrefixes);
+    const newSet = new Set(_prefixes);
+
     for (let prefix of _prefixes) {
-        console.log("Prefix:", prefix);
+        if (!oldSet.has(prefix)) {
+            console.log(green(`Prefix: ${prefix}`));
+        } else {
+            console.log(`Prefix: ${prefix}`);
+        }
     }
+
+    for (let prefix of oldPrefixes) {
+        if (!newSet.has(prefix)) {
+            console.log(red(`Prefix: ${prefix}`));
+        }
+    }
+
     await prefixes.set(key, { prefixes: _prefixes });
 }

package/src/0-path-value-core/ValidStateComputer.ts

@@ -50,18 +50,12 @@ class ValidStateComputer {
                     let parentPath = getParentPathStr(value.path);
                     // We check for the parent trigger late because getting the parent path is a little bit expensive
                     if (!config.initialTriggers.parentPaths.has(parentPath)) {
-                        if (PathRouter.isLocalPath(value.path)) {
-                            require("debugbreak")(2);
-                            debugger;
-                        }
-
-                        console.log(`Ignoring duplicate value.`, {
+                        console.info(`Ignoring duplicate value.`, {
                             path: value.path,
                             timeId: value.time.time,
                             timeIdVersion: value.time.version,
                             timeIdFull: value.time,
-                            existingValid: existingValue?.valid,
-                            newValid: value.valid,
+                            valid: value.valid,
                         });
                         // NOTE: This should be safe. If this causes any problems, it will probably be if we're not the authority on it. In which case, we could move this check so it only happens if we are the authority on it. However, I think even if we aren't the authority, if we already have the exact value and the valid state hasn't changed, we really shouldn't have to ingest it. Watchers should be triggering when they start watching, so they shouldn't need us to trigger them again by re-ingesting it. 
                         return false;

package/src/2-proxy/schema2.ts

@@ -150,6 +150,19 @@ type TypeDefType<T = never, Optional = false> = {
         schema: NewT
     ): TypeDefType<{ [key: string | number]: TypeDefToT<NewT> }>;
 
+    /** Lookup with GC enabled, but excluded from root lookup routing.
+     *      Behaves like a regular lookup but won't be used as a prefix for sharding.
+     */
+    lookupNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: string]: TypeDefToT<NewT> }>;
+    lookupNumberNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: number]: TypeDefToT<NewT> }>;
+    lookupEitherNoRoot<NewT>(
+        schema: NewT
+    ): TypeDefType<{ [key: string | number]: TypeDefToT<NewT> }>;
+
 
 
     // TODO: Support .union, which just spreads the inputs together, making an object which
@@ -247,22 +260,42 @@ export class Schema2Fncs {
         return schema2ToTypeString(schema);
     }
 
-    /** Uses * for wildcards */
-    public static getLookups(schema: Schema2): string[][] {
-        let internalType = typeDefCached(schema);
-        let lookupPaths = new Set<string>();
-
-        for (let def of internalType.defs) {
-            for (let i = 0; i < def.path.length; i++) {
-                if (def.path[i] === WildCard) {
-                    let lookupPath = def.path.slice(0, i);
-                    let stringPath = lookupPath.map(p => p === WildCard ? "*" : p as string);
-                    lookupPaths.add(JSON.stringify(stringPath));
+    public static getRootLookups(schema: Schema2): string[][] {
+        let defs = typeDefCached(schema).defs.slice();
+
+        sort(defs, x => x.path.length);
+
+        let rootLookups = new Set<string>();
+        for (let def of defs) {
+            if (def.isLookupNoRoot) continue;
+            if (!def.isObjectGC) continue;
+            let path = def.path;
+            if (path.at(-1) !== WildCard) continue;
+            path = path.slice(0, -1);
+
+            let pathParts = path.map(p => p === WildCard ? "" : p);
+            let pathKey = JSON.stringify(pathParts);
+
+            let hasAncestor = false;
+            for (let i = 0; i < path.length - 1; i++) {
+                let ancestorPath = path.slice(0, i + 1);
+                let ancestorParts = ancestorPath.map(p => p === WildCard ? "" : p);
+                let ancestorKey = JSON.stringify(ancestorParts);
+                if (rootLookups.has(ancestorKey)) {
+                    hasAncestor = true;
+                    break;
                 }
             }
+
+            if (!hasAncestor) {
+                rootLookups.add(pathKey);
+            }
         }
 
-        return Array.from(lookupPaths).map(p => JSON.parse(p));
+        return Array.from(rootLookups).map(p => {
+            let parts = JSON.parse(p) as string[];
+            return parts;
+        });
     }
 }
 
@@ -518,6 +551,8 @@ type InternalTypeDef = {
     isBoolean?: boolean;
     isNull?: boolean;
 
+    isLookupNoRoot?: boolean;
+
     gcDelay?: number;
 
     defaultValue?: unknown;
@@ -544,6 +579,7 @@ function typeDefTypeToInternalType(
     }
     let isLookup = false;
     let isLookupGC = false;
+    let isLookupNoRoot = false;
     if (typeDef.path.includes("lookup") || typeDef.path.includes("lookupNumber") || typeDef.path.includes("lookupEither")) {
         isLookup = true;
         isLookupGC = true;
@@ -552,6 +588,11 @@ function typeDefTypeToInternalType(
         isLookup = true;
         isLookupGC = false;
     }
+    if (typeDef.path.includes("lookupNoRoot") || typeDef.path.includes("lookupNumberNoRoot") || typeDef.path.includes("lookupEitherNoRoot")) {
+        isLookup = true;
+        isLookupGC = true;
+        isLookupNoRoot = true;
+    }
 
     let isObject = false;
     let isObjectGC = false;
@@ -594,6 +635,7 @@ function typeDefTypeToInternalType(
         // NOTE: Gets reset if the nestedValues are used with a primitive
         nestedValues.isNotAtomic = true;
         nestedValues.isObjectGC = isObjectGC || isLookupGC;
+        nestedValues.isLookupNoRoot = isLookupNoRoot;
         nestedValues.gcDelay = Number(typeDef.pathWithCalls.find(x => x.key === "gcDelay")?.callParams?.[0]) || undefined;
 
 

package/src/3-path-functions/syncSchema.ts

@@ -372,27 +372,11 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
                 };
             }
 
-            // Register all root lookups as prefixes.
             let dataRoot = getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Data);
-            let paths = Schema2Fncs.getLookups(schema).map(x =>
-                joinPathStres(dataRoot, getPathStr(x))
-            );
-            // NOTE: This will break with wildcards, but... we also take only the highest level lookups, so it shouldn't be an issue
-            sort(paths, x => x.length);
-            let rootLookups = new Set<string>();
-            for (let path of paths) {
-                let parts = getPathFromStr(path);
-                // Don't add if any of our ancestors are added
-                function hasAncestor() {
-                    for (let i = 0; i < parts.length - 1; i++) {
-                        let ancestorPath = getPathStr(parts.slice(0, i));
-                        if (rootLookups.has(ancestorPath)) return true;
-                    }
-                }
-                if (!hasAncestor()) {
-                    rootLookups.add(path);
-                    prefixes.push(path);
-                }
+            let rootLookups = Schema2Fncs.getRootLookups(schema);
+            for (let lookup of rootLookups) {
+                let path = joinPathStres(dataRoot, getPathStr(lookup));
+                prefixes.push(path);
             }
         }
 

package/src/4-deploy/deployPrefixes.ts

@@ -7,6 +7,9 @@ async function main() {
     await import(deployPath);
 
     let prefixes = await getPrefixesForDeploy();
+    for (let prefix of prefixes) {
+        console.log(prefix);
+    }
     await setShardPrefixes(prefixes);
 }
 

package/src/diagnostics/SyncTestPage.tsx

@@ -16,6 +16,8 @@ import { sort } from "socket-function/src/misc";
 import { PathAuditerController } from "./pathAuditer";
 import { t } from "../2-proxy/schema2";
 
+//todonext
+// So... We need to double write, to write to multiple keys at once, in something that's not a lookup, or no, is a lookup, but before we deploy, so it doesn't have the proper or no, a nested lookup, even better. 
 
 let { data, functions } = Querysub.createSchema({
     values: t.lookup(t.number)

package/test.ts

@@ -16,6 +16,7 @@ import { ClientWatcher } from "./src/1-path-client/pathValueClientWatcher";
 import { RemoteWatcher } from "./src/1-path-client/RemoteWatcher";
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { shutdown } from "./src/diagnostics/periodic";
+import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
 
 let tempTestSchema = Querysub.createSchema({
     value: t.number,
@@ -27,7 +28,12 @@ let tempTestSchema = Querysub.createSchema({
 });
 
 async function main() {
-    await Querysub.hostService("test");
+
+    let prefixes = await getShardPrefixes();
+    for (let prefix of prefixes) {
+        console.log(prefix);
+    }
+    // await Querysub.hostService("test");
 
     // let path = getProxyPath(() => tempTestSchema.data().value);
     // console.log({ path });
@@ -40,14 +46,14 @@ async function main() {
     // ClientWatcher.DEBUG_WRITES = true;
     // RemoteWatcher.DEBUG = true;
 
-    let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
-    console.log({ value });
-    await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
-    await delay(3000);
-    let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
-    console.log({ value2 });
+    // let value = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    // console.log({ value });
+    // await Querysub.commitAsync(() => tempTestSchema.data().value++, { doNotStoreWritesAsPredictions: true });
+    // await delay(3000);
+    // let value2 = await Querysub.commitAsync(() => tempTestSchema.data().value);
+    // console.log({ value2 });
 
-    await shutdown();
+    // await shutdown();
 
     // let test = await Querysub.commitAsync(() => {
     //     let live = deploySchema()[getDomain()].deploy.live.hash;