querysub 0.461.0 → 0.463.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.461.0",
+  "version": "0.463.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-d-trust/NetworkTrust2.ts

@@ -13,7 +13,7 @@ import debugbreak from "debugbreak";
 import { devDebugbreak, getDomain, isDevDebugbreak, isPublic, isRecovery } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
 import { runInSerial } from "socket-function/src/batching";
-import { Querysub } from "../4-querysub/QuerysubController";
+import { Querysub } from "../4-querysub/Querysub";
 import { magenta } from "socket-function/src/formatting/logColors";
 
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing

package/src/0-path-value-core/LockWatcher2.ts

@@ -6,10 +6,6 @@ import { MAX_CHANGE_AGE, Time, PathValue, byLockGroup, compareTime } from "./pat
 import { PathRouter } from "./PathRouter";
 import { getInternalValueWatchPrefix, pathWatcher } from "./PathWatcher";
 
-function isGoldenLock(time: number, now: number): boolean {
-    return time < now - MAX_CHANGE_AGE * 2;
-}
-
 function getLockWatcher2NodeId() {
     return getInternalValueWatchPrefix() + "_LockWatcher2";
 }
@@ -63,8 +59,9 @@ class LockWatcher2 {
         if (values.length === 0) return;
         this.ensureGarbageCollectLoop();
         let lockGroups = byLockGroup(values);
+        let threshold = now - MAX_CHANGE_AGE;
         for (let [locks, valueGroup] of lockGroups) {
-            locks = locks.filter(x => !isGoldenLock(x.endTime.time, now));
+            locks = locks.filter(x => x.endTime.time > threshold);
             if (locks.length === 0) continue;
             for (let lock of locks) {
                 if (!PathRouter.isSelfAuthority(lock.path)) {

package/src/0-path-value-core/PathRouter.ts

@@ -13,6 +13,8 @@ import { rangesOverlap, removeRange } from "../rangeMath";
 import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
 import { getBufferInt } from "../bits";
 
+import { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH } from "./PathRouterConstants";
+export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 
 // Cases
 // 1) Whole path hash
@@ -22,9 +24,6 @@ import { getBufferInt } from "../bits";
 
 
 
-export const LOCAL_DOMAIN = "LOCAL";
-export const LOCAL_DOMAIN_PATH = getPathStr1(LOCAL_DOMAIN);
-
 // NOTE: The goal is for a user on the site to require talking to as few authorities as possible. Because most things are nested in lookups, if we have a prefix per root lookup, this should mean if you're on a page, most of the data should be talking to the same authority, as most of the data should have the exact same hash.
 //  - We also support the exclude default case, which allows you to only handle certain prefixes, so you can make a function only run on certain servers.
 export type AuthoritySpec = {

package/src/0-path-value-core/PathRouterConstants.ts

@@ -0,0 +1,4 @@
+import { getPathStr1 } from "../path";
+
+export const LOCAL_DOMAIN = "LOCAL";
+export const LOCAL_DOMAIN_PATH = getPathStr1(LOCAL_DOMAIN);

package/src/0-path-value-core/PathValueCommitter.ts

@@ -123,7 +123,6 @@ class PathValueCommitter {
             }
         }
 
-        // NOTE: This function will just ignore writes we aren't an authority on, so we can just give it everything.
         validStateComputer.ingestValuesAndValidStates({
             pathValues: pathValuesToIngest,
             parentSyncs: [],

package/src/0-path-value-core/PathValueController.ts

@@ -323,7 +323,7 @@ export class PathValueControllerBase {
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
         let values: PathValue[] = [];
         for (let entry of entries) {
-            let value = authorityStorage.getValueExact(entry.path, entry.time);
+            let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
             if (!value) continue;
             if (!value.valid) continue;
             if (value.isTransparent) continue;

package/src/0-path-value-core/PathWatcher.ts

@@ -87,6 +87,9 @@ class PathWatcher {
         parents: Set<string>;
         fullHistory?: boolean;
     }>());
+    public debugGetWatcherPaths(nodeId: string) {
+        return this.watchersToPaths.get(nodeId);
+    }
 
     /** Automatically watches the remote if it's a remote path.
      *  Automatically unwatches the remote when the paths no longer have any callbacks.
@@ -346,7 +349,7 @@ class PathWatcher {
         valuesChanged: Set<PathValue>;
 
         // Mutates initialTriggers.values
-        initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+        initialTriggers: { values: Set<string>; parentPaths: Set<string>; initialTriggerNonHistoryWatchers?: Set<string> };
 
         onlyTriggerNodeId?: string;
     }) {
@@ -365,6 +368,9 @@ class PathWatcher {
         for (let path of initialTriggers?.values ?? []) {
             triggerPaths.add(path);
         }
+        for (let path of initialTriggers?.initialTriggerNonHistoryWatchers ?? []) {
+            triggerPaths.add(path);
+        }
 
         let valuePerPath = keyByArray(Array.from(valuesChanged), x => x.path);
 
@@ -409,6 +415,7 @@ class PathWatcher {
 
             // Important. We can't do the initial trigger if we aren't synced. If we're the authority, this doesn't matter. This is only important for proxy watchers. 
             let isInitialTrigger = !!initialTriggers?.values.has(path) && authorityStorage.isSynced(path);
+            let isInitialTriggerNonHistory = !!initialTriggers?.initialTriggerNonHistoryWatchers?.has(path);
             const triggerNodeChanged = (watcher: NodeId) => {
                 if (onlyTriggerNodeId && watcher !== onlyTriggerNodeId) return;
                 let changes = changedPerCallbacks.get(watcher);
@@ -421,12 +428,14 @@ class PathWatcher {
                 for (let value of values) {
                     changes.values.add(value);
                 }
-                if (isInitialTrigger) {
+                let watcherObj = this.watchersToPaths.get(watcher);
+                let fullHistory = watcherObj?.fullHistory;
+                if (isInitialTrigger || isInitialTriggerNonHistory && !fullHistory) {
                     changes.initialTriggers.values.add(path);
-                    let watcherObj = this.watchersToPaths.get(watcher);
-                    let fullHistory = watcherObj?.fullHistory;
                     if (fullHistory) {
                         let history = authorityStorage.getValuePlusHistory(path);
+                        // There's no point in sending them invalid values. This is going to clobber any existing values. So if they had a value that was valid and it's now invalid, if we just don't send it, they'll remove it (and removal literally means setting the valid state to false, so it's identical).
+                        history = history.filter(x => x.valid);
                         for (let historicalValue of history) {
                             changes.values.add(historicalValue);
                         }
@@ -435,7 +444,7 @@ class PathWatcher {
                         }
                     } else {
                         // NOTE: We won't be given a value if it isn't synced, which will be never because we check if we're synced above. However, the type system doesn't know that. 
-                        let value = authorityStorage.getValueAtTime(path);
+                        let value = authorityStorage.getValueAtOrBeforeTime(path);
                         if (value) {
                             changes.values.add(value);
                         }

package/src/0-path-value-core/ValidStateComputer.ts

@@ -5,31 +5,25 @@ import { isDiskAudit } from "../config";
 import { getParentPathStr, getPathFromStr } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
-import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE } from "./pathValueCore";
+import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE, createMissingEpochValue, Time, MISSING_TRANSACTION_PART_TIMEOUT, isOurPrediction, DEFER_LOCK_WINDOW } from "./pathValueCore";
 import { pathWatcher } from "./PathWatcher";
 import { lockWatcher2 } from "./LockWatcher2";
 import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
+import { isMissingTransactionPart } from "../2-proxy/TransactionDelayer";
+import { formatTime } from "socket-function/src/formatting/format";
+import { isDefined } from "../misc";
 
 class ValidStateComputer {
 
-    private capturePrevValidStates(valuePaths: PathValue[]): Map<PathValue, boolean | undefined> {
-        let prevValidStates = new Map<PathValue, boolean | undefined>();
-        for (let value of valuePaths) {
-            let prevValidState = authorityStorage.getValueAtTime(value.path, value.time)?.valid;
-            prevValidStates.set(value, prevValidState);
-        }
-        return prevValidStates;
-    }
-
     @measureFnc
     public ingestValuesAndValidStates(config: {
         pathValues: PathValue[];
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
-
     }) {
-        let { pathValues, parentSyncs, initialTriggers, doNotArchive } = config;
+        let { pathValues, parentSyncs, doNotArchive } = config;
+        let initialTriggers = { ...config.initialTriggers, initialTriggerNonHistoryWatchers: new Set<string>() };
 
         // TODO: We might want to add back optimizations for "no watches and no locks"?
         // TODO: If we find it is a serious performance problem, we could just send the valid state instead of sending the full path value and values are rejected (this is what we used to do).
@@ -38,15 +32,47 @@ class ValidStateComputer {
 
         authorityStorage.addParentSyncs(parentSyncs);
 
+        // Dedup by (path, time): if multiple values for the same (path, time) arrive in this batch,
+        //  keep only the last one. Otherwise the storage-vs-incoming dedup below can drop the wrong one
+        //  (e.g. a correction whose valid state happens to match stale storage gets filtered, while the
+        //  earlier incorrect value in the same batch survives and clobbers storage).
+        {
+            let byPath = new Map<string, PathValue[]>();
+            for (let value of pathValues) {
+                let arr = byPath.get(value.path);
+                if (!arr) {
+                    arr = [];
+                    byPath.set(value.path, arr);
+                }
+                let index = binarySearchIndex(arr.length, i => compareTime(arr![i].time, value.time));
+                if (index >= 0) {
+                    arr[index] = value;
+                } else {
+                    arr.splice(~index, 0, value);
+                }
+            }
+            let deduped: PathValue[] = [];
+            for (let arr of byPath.values()) {
+                for (let v of arr) {
+                    deduped.push(v);
+                }
+            }
+            deduped.sort((a, b) => compareTime(a.time, b.time));
+            pathValues = deduped;
+        }
+
         let ourValues: PathValue[] = [];
         let notOurValues: PathValue[] = [];
+        // Split into our values and not our values
+        //  - Also Ignore values we already received with no valid state change. Required to prevent authorities from infinitely looping while exchanging values.
         pathValues = pathValues.filter(value => {
             // NOTE: I think if it's initial trigger we need to process it anyways?
             if (!config.initialTriggers.values.has(value.path)) {
                 // Ignore values we already have. We receive duplicates, often due to values being broadcast multiple times in order to prevent data loss.
-                let existingValue = authorityStorage.getValueExact(value.path, value.time);
+                let existingValue = authorityStorage.getValueExactMaybeRejected(value.path, value.time);
                 // If we already have the value and the valid state is the exact same, then remove it from path values and don't even add it to our values. Pretend like we never received it.
-                if (existingValue && !!existingValue.valid === !!value.valid) {
+                // NOTE: We don't do any coercion here because we want the undefined state to not equal the false state. 
+                if (existingValue && existingValue.valid === value.valid) {
                     let parentPath = getParentPathStr(value.path);
                     // We check for the parent trigger late because getting the parent path is a little bit expensive
                     if (!config.initialTriggers.parentPaths.has(parentPath)) {
@@ -70,12 +96,37 @@ class ValidStateComputer {
             }
             return true;
         });
-        authorityStorage.ingestValues(notOurValues, { doNotArchive });
+
+        // NOTE: Watch value locks, will internally watch remote locks if we aren't the authority on the values
+        lockWatcher2.watchValueLocks(ourValues, now);
+
+        // ALL of the direct values are assumed changed
+        let valuesChanged = new Set<PathValue>(pathValues);
+        // ONLY Put values that we are the owner of in this. As for the other ones, we just ingest the updates, but we don't calculate their valid state.
+        let dependenciesChanged = new Set<PathValue>(ourValues);
+        // AND, Even if they're not our values, we may still be watching them. But then we need to trigger the actual watcher, not the not our value. All watchers are owned by us, but the things that are watched, as in the observed values, might not be owned by us. 
+        //  - AND, Inside of the loop, after we update a value, we also find all the watchers for it. So we're essentially doing one step of the loop early here. That way we don't have to compute the valid state values for ourselves. We short short-circuit that because we're not the owner. But for values that we do own, we do compute the valid state, and then we find the watcher. 
+        for (let value of notOurValues) {
+            let watchers = lockWatcher2.getValuePathWatchers(value);
+            for (let watcher of watchers) {
+                dependenciesChanged.add(watcher);
+            }
+
+            if (!value.valid) {
+                initialTriggers.initialTriggerNonHistoryWatchers.add(value.path);
+            }
+        }
 
 
-        // Use initial triggers, at least for all the values that we aren't the authority of, to clobber the old values, creating new shallow copies of the old values with a valid state false if we don't have the corresponding value in our new values, including for all parent paths. 
+        // NOTE: The code should still work if we have duplicate PathValues (same path, same time, but !==). It can be less efficient, but it should still work because this happens in certain cases. 
+        let initialPrevValidStates = new Map<PathValue, boolean | undefined>();
+        for (let value of dependenciesChanged) {
+            let prevValidState = authorityStorage.getValueExactMaybeRejected(value.path, value.time)?.valid;
+            initialPrevValidStates.set(value, prevValidState);
+        }
+
+        // Create valid state false for old values on initial sync paths (ONLY for notOurValues)
         {
-            // TODO: This code is similar to the code in trigger values change, but it's also dealing with watchers and other stuff, so I think it makes sense to have it also here.
             let fullInitialPaths = new Set<string>();
             for (let path of initialTriggers?.values ?? []) {
                 fullInitialPaths.add(path);
@@ -92,89 +143,101 @@ class ValidStateComputer {
             let fullInitialPathsArr = Array.from(fullInitialPaths);
             fullInitialPathsArr = fullInitialPathsArr.filter(x => !PathRouter.isSelfAuthority(x));
 
+            // Remove all other values (except for predictions)
+            // NOTE: We do this before we ingest our values. And the caller makes sure that the values that we have are actually came after this initial sync on the wire or are part of the initial sync, or are the latest, regardless. So, we don't need to worry about issues there, such as not clobbering values that we're about to ingest. That's fine. 
+
+
             let valueByPath = keyByArray(notOurValues, x => x.path);
+
+
             for (let path of fullInitialPathsArr) {
+                if (PathRouter.isSelfAuthority(path)) {
+                    console.error(`Try to apply an initial sync of a value that we're the authority on. This should have been filtered out much earlier. Path: ${path}`);
+                    continue;
+                }
                 let newValues = valueByPath.get(path) ?? [];
-                let prevValues = authorityStorage.getValuePlusHistory(path);
-
-                newValues.sort((a, b) => compareTime(a.time, b.time));
 
+                let prevValues = authorityStorage.getValuePlusHistory(path);
                 for (let value of prevValues) {
-                    let newIndex = binarySearchIndex(newValues.length, i => compareTime(newValues[i].time, value.time));
-                    if (newIndex < 0) {
-                        let newValue = newValues.at(-1);
-                        console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
-                            path: value.path,
-                            timeId: value.time.time,
-                            timeIdFull: value.time,
-                            newTimeId: newValue?.time.time,
-                            newTimeIdFull: newValue?.time,
-                        });
-                        ourValues.push({ ...value, valid: false, });
-                    }
+                    if (!value.valid) continue;
+                    // Don't remove predictions. They'll remove themselves anyway. They always get clobbered by a server, write, and if they don't get clobbered, they have a timeout. If we remove predictions, then cases where we're constantly writing to new paths are really annoying and start to glitch out on the client, where we predict it, we remove it, and then we add it back. 
+                    //      NOTE: If not removing predictions breaks things, we could probably remove predictions, at least when isServer().
+                    if (isOurPrediction(value)) continue;
+
+                    // NOTE: This isn't required for correctness, it just makes our logs a lot easier to read. 
+                    if (newValues.some(x => compareTime(x.time, value.time) === 0)) continue;
+
+                    // NOTE: If it's one of the new values, ingest values will change the valid state anyway, so it's fine to set it here. 
+                    console.info(`Rejecting past value due to initial sync (might be immediately set again): ${debugPathValuePath(value)}`, {
+                        path: value.path,
+                        timeId: value.time.time,
+                        timeIdFull: value.time,
+                    });
+                    initialPrevValidStates.set(value, true);
+                    // @ts-expect-error
+                    value.valid = false;
+                    valuesChanged.add(value);
                 }
             }
         }
 
-        let initialPrevValidStates: Map<PathValue, boolean | undefined> | undefined = this.capturePrevValidStates(ourValues);
+        authorityStorage.ingestValues(notOurValues, { doNotArchive });
         authorityStorage.ingestValues(ourValues, { doNotArchive });
 
-        // NOTE: Watch value locks, will internally watch remote locks if necessary. 
-        lockWatcher2.watchValueLocks(ourValues, now);
-
-        // NOTE: Initial values always get put into computeValidStates, which then ingests, but does not compute values we are not the authority for.
-        let valuesChanged = new Set<PathValue>(pathValues);
-        let dependenciesChanged = new Set<PathValue>(ourValues);
-        let alreadyTriggered = new Set<PathValue>(dependenciesChanged);
-        // NOTE: For our own values, we don't care what the input valid state is, compute valid states will calculate if it's changed. 
-        for (let value of notOurValues) {
-            // NOTE: We could be smarter and determine if the value we're receiving has a valid state change. However, it's probably fine. 
-            // NOTE: Any watchers will get their valid states computed. I think that's really the only reason to watch something is if we own it and want to compute its valid state. 
-            let watchers = lockWatcher2.getValuePathWatchers(value);
-            for (let watcher of watchers) {
-                if (alreadyTriggered.has(watcher)) continue;
-                dependenciesChanged.add(watcher);
-                alreadyTriggered.add(watcher);
-            }
-        }
 
         let loopCount = 0;
+        let deferredAll = new Set<PathValue>();
 
         while (dependenciesChanged.size > 0) {
             loopCount++;
             if (loopCount > 3000) {
+                require("debugbreak")(2);
+                debugger;
                 console.error("Too many loops in valid state loop. aborting.", {
                     loopCount,
                     changedCount: dependenciesChanged.size,
-                    totalTriggered: alreadyTriggered.size,
                     exampleChanged: Array.from(dependenciesChanged)[0].path,
                 });
                 break;
             }
             // Just because the dependencies change doesn't mean the valid state has changed. We actually have to compute to see if it has changed. Of course, if we're not the authority, we won't compute the valid site at all. It'll just ignore it, which is good. If we're not the authority, we don't want to be doing recursive computation. Because we prepopulate it, the values change with all of the input values, we'll still get a trigger. We just won't get a nested trigger.
-            let validStateChanged = validStateComputer.computeValidStates(
+            let result = validStateComputer.computeValidStates(
                 Array.from(dependenciesChanged),
                 now,
                 initialPrevValidStates,
-            ).changed;
-            initialPrevValidStates = undefined;
+            );
+            let validStateChanged = result.changed;
+            for (let d of result.deferred) {
+                deferredAll.add(d);
+            }
+            for (let change of validStateChanged) {
+                initialPrevValidStates.set(change, change.valid);
+                deferredAll.delete(change);
+            }
             dependenciesChanged.clear();
 
             for (let validStateChange of validStateChanged) {
                 valuesChanged.add(validStateChange);
-                // IMPORTANT! The consequence of this is that if someone predicts a value, sends it to us, it's not valid, we will see it's not valid, and we will tell all of the nodes that are watching that path about all of the values on that path. This is inefficient. However, rejected values should be quite rare, so it shouldn't be too inefficient. 
+
+                // NOTE: This approach should be faster than doing a full initial sync, but we'll see if it breaks anything.
                 if (!validStateChange.valid) {
-                    // If it's not valid, we need to send the next available valid value. But we haven't done all the fullness computation yet, so the current next available valid value might be recomputed, it might be invalid. So we'll just do the initial trigger. It's not going to be that expensive as the GCing should mean that most paths have a small history, and rejections should be quite rare. 
-                    initialTriggers.values.add(validStateChange.path);
+                    initialTriggers.initialTriggerNonHistoryWatchers.add(validStateChange.path);
                 }
+
+                // // NOTE: As of right now, I think the only reason to force an initial trigger in this case is so that we can bust all the values through the query sub http server to the clients. Because if we just tell the HTTP server that one value has been rejected, it can go tell its clients, but that won't give them the latest value. Which isn't great. We should really have something where we can tell them. HMM...
+                // // IMPORTANT! The consequence of this is that if someone predicts a value, sends it to us, it's not valid, we will see it's not valid, and we will tell all of the nodes that are watching that path about all of the values on that path. This is inefficient. However, rejected values should be quite rare, so it shouldn't be too inefficient. 
+                // if (!validStateChange.valid) {
+                //     // NOTE: TECHNICALLY, We could restrict this change in the case when a node just created the value, so we haven't told anyone about it. However, also, this is replacing the case where we send that update to all the other nodes. And so, if we do the math, let's say we have a history window of 100 seconds, and 1% of the writes get rejected, and we're writing once per second. Then this would result in sending twice as many values in total, which isn't great, but also a 1% rejection rate is pretty high. 
+                //     initialTriggers.values.add(validStateChange.path);
+                // }
             }
             // And then for everything that had a valid state change, go check if it has a dependency that was watching it.
+            // NOTE: We used to make sure we don't re-trigger something that triggered before, which prevents always running computeValidStates twice on new values. BUT... it causes issues if you have out of order evaluation. We try to fix out of order evaluation in computeValidStates, but I think locks could be setup so it's not possible. So instead... we just allow evaluating values multiple times, which is twice as slow, but... it's probably not a bottleneck.
+            //   IF we change this, test with a simple x++ update, where the function has no shard hints, to verify it doesn't break (queue about 100 events at once to test)
             for (let value of validStateChanged) {
                 let watchers = lockWatcher2.getValuePathWatchers(value);
                 for (let watcher of watchers) {
-                    if (alreadyTriggered.has(watcher)) continue;
                     dependenciesChanged.add(watcher);
-                    alreadyTriggered.add(watcher);
                 }
             }
         }
@@ -183,13 +246,40 @@ class ValidStateComputer {
             valuesChanged,
             initialTriggers
         });
+
+        // If any lockGroups deferred decision (waiting on a value from a foreign authority that
+        //  may still be in flight), schedule a re-evaluation after DEFER_LOCK_DELAY. By then the
+        //  age check in isLockValid will treat any still-missing locks as definitively rejected.
+        if (deferredAll.size > 0) {
+            let deferredArr = Array.from(deferredAll);
+            let deferredPaths = new Set<string>();
+            for (let v of deferredArr) {
+                deferredPaths.add(v.path);
+            }
+            setTimeout(() => {
+                this.ingestValuesAndValidStates({
+                    pathValues: deferredArr,
+                    parentSyncs: [],
+                    // Add deferred paths to initialTriggers.values to bypass the storage-vs-incoming
+                    //  dedup (which would otherwise filter these out since storage already has them).
+                    initialTriggers: { values: deferredPaths, parentPaths: new Set() },
+                });
+            }, DEFER_LOCK_WINDOW);
+        }
     }
 
-    private isLockValid(lock: ReadLock): boolean {
-        let value = authorityStorage.getValueAtTime(lock.path, lock.endTime);
+    private isLockValid(lock: ReadLock, now: number): true | false | "defer" {
+        if (lock.readIsTransparent) return true;
+        let value = authorityStorage.getValueExactMaybeRejected(lock.path, lock.startTime);
         if (!value) {
             // If not synced, we don't want to reject it yet
-            return !authorityStorage.isSynced(lock.path);
+            if (!authorityStorage.isSynced(lock.path)) return true;
+            // Value missing on a foreign-authority path. It may still be in flight; defer the decision
+            //  until DEFER_LOCK_DELAY has elapsed since the lock's startTime.
+            if (!PathRouter.isSelfAuthority(lock.path) && now - lock.startTime.time < DEFER_LOCK_WINDOW) {
+                return "defer";
+            }
+            return false;
         }
         return !!value.valid;
     }
@@ -219,55 +309,115 @@ class ValidStateComputer {
     public computeValidStates(
         valuePaths: PathValue[],
         now: number,
-        prevValidStates = this.capturePrevValidStates(valuePaths)
-    ): { changed: PathValue[] } {
+        prevValidStates: Map<PathValue, boolean | undefined>,
+    ): { changed: PathValue[]; deferred: PathValue[] } {
         let changed: PathValue[] = [];
+        let deferred: PathValue[] = [];
 
         let lockGroups = byLockGroup(valuePaths);
 
+        // Sort by the lock time to remove unneeded cycles (although we still always loop at least once).
+        let entries = Array.from(lockGroups);
+        entries = entries.filter(x => x[0].length > 0);
+        entries = entries.sort((a, b) => compareTime(a[0][0].endTime, b[0][0].endTime));
+        lockGroups = new Map(entries);
+
         for (let [locks, valueGroup] of lockGroups) {
             if (valueGroup.length === 0) continue;
+            if (locks.length === 0) continue;
 
-            let valid = true;
-            for (let lock of locks) {
-                let age = now - lock.endTime.time;
-                // If it's old enough, then it's always valid, we can't change the valid save something after its max change age, even if it's invalid. 
-                if (age > MAX_CHANGE_AGE) continue;
 
-                if (!this.isLockValid(lock)) {
-                    onPathInteracted(lock.path, 2);
+            let valid: true | false | "defer" = true;
+
+            let lockTime = locks[0].endTime;
+            for (let lock of locks) {
+                if (compareTime(lock.endTime, lockTime) !== 0) {
+                    require("debugbreak")(2);
+                    debugger;
+                    console.error(`Lock has inconsistent end times. ${debugTime(lock.endTime)} !== ${debugTime(lockTime)}. Path: ${lock.path}`);
                     valid = false;
-                    this.logLockInvalid(valueGroup, lock, now);
-                    break;
                 }
-                let contention = this.isLockContentionFree(lock);
-                if (contention) {
-                    onPathInteracted(lock.path, 2);
+            }
+
+            for (let value of valueGroup) {
+                if (!PathRouter.isSelfAuthority(value.path)) {
+                    require("debugbreak")(2);
+                    debugger;
+                    console.error(`Somehow we're attempting to compute the valid state of a value we don't own. This should be impossible. ${debugPathValue(value)}`);
                     valid = false;
-                    this.logLockContention(valueGroup, lock, contention, now);
-                    break;
                 }
             }
+
+            let invalidReason: string | undefined;
+            let age = now - lockTime.time;
+            // If it's old enough, then it's always valid, we can't change the valid save something after its max change age, even if it's invalid.
+            if (age < MAX_CHANGE_AGE && valid === true) {
+                for (let lock of locks) {
+                    let lockResult = this.isLockValid(lock, now);
+                    if (lockResult === "defer") {
+                        valid = "defer";
+                        // Don't break — a later lock might be definitively false, which beats defer.
+                        continue;
+                    }
+                    if (!lockResult) {
+                        onPathInteracted(lock.path, 2);
+                        valid = false;
+                        let lockValue = authorityStorage.getValueExactMaybeRejected(lock.path, lock.startTime);
+                        invalidReason = `lock-invalid on ${lock.path} at ${debugTime(lock.startTime)} (value=${lockValue ? debugPathValue(lockValue) : "missing"}, synced=${authorityStorage.isSynced(lock.path)}, transparent=${lock.readIsTransparent})`;
+                        this.logLockInvalid(valueGroup, lock, now);
+                        break;
+                    }
+                    let contention = this.isLockContentionFree(lock);
+                    if (contention) {
+                        onPathInteracted(lock.path, 2);
+                        valid = false;
+                        invalidReason = `lock-contention on ${lock.path} range ${debugTime(lock.startTime)}..${debugTime(lock.endTime)} conflicts=[${contention.map(x => debugTime(x.time)).join(", ")}]`;
+                        this.logLockContention(valueGroup, lock, contention, now);
+                        break;
+                    }
+                }
+
+                // NOTE: It's pretty easy for us to look at all the valid path values for the locks and then see if there's an inconsistent transaction. However, I don't think this is a common case. I think the common case is were depending on just a single write, which happened to land on a different server than us.
+            }
+
+            if (valid === "defer") {
+                // Don't decide; leave pathValue.valid and storage alone. Schedule retry at outer level.
+                for (let pathValue of valueGroup) {
+                    deferred.push(pathValue);
+                }
+                continue;
+            }
+
             for (let pathValue of valueGroup) {
                 let prevValidState = prevValidStates.get(pathValue);
+
                 // IMPORTANT! This means if it didn't previously exist and it's presently rejected, we count that as a change, as it this is going from undefined to false. This is actually very useful, as a lot of places want to know if a write is rejected, so they can display it in the UI for the developer.
                 if (valid === prevValidState) continue;
 
                 changed.push(pathValue);
 
-                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it. 
-                (pathValue as any).valid = valid;
+                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it.
+                // @ts-expect-error
+                pathValue.valid = valid;
+
+                // HACK: There are times when we might be evaluating the same path value multiple times at once (ex, multiple initial syncs at once). In which case we might not equal the stored path value, so we need to forcefully update the valid state of it.
+                let storedPathValue = authorityStorage.getValueExactMaybeRejected(pathValue.path, pathValue.time);
+                // I think this is fine. I think it happens if we receive it multiple times due to sharding. It's a bit inefficient, but... SHOULD be fine.
+                if (storedPathValue && pathValue !== storedPathValue) {
+                    // @ts-expect-error
+                    storedPathValue.valid = valid;
+                }
             }
         }
 
-        return { changed };
+        return { changed, deferred };
     }
 
     private logLockInvalid(valueGroup: PathValue[], lock: ReadLock, now: number) {
         // This is good... unless it happens A LOT. Then the app needs to change (unless it
         //  is a game, or something with realtime competition, in which case this is probably unavoidable).
         if (valueGroup.length > 0 && lock.endTime.version !== Number.MAX_SAFE_INTEGER && lock.startTime.version !== -2) {
-            if ((!isCoreQuiet || !isNode()) && debugRejections) {
+            if (true) {
                 let timeToReject = now - valueGroup[0].time.time;
                 let message = `!!! (VALUE REJECTED) VALUE REJECTED DUE TO USING MISSING / REJECTED READ!!!(rejected after ${timeToReject}ms at ${Date.now()})`;
                 message += `\n${debugTime(valueGroup[0].time)} (write)`;
@@ -307,9 +457,7 @@ class ValidStateComputer {
 
             let changed = valueGroup.filter(x => x.valid);
             if (changed.length > 0) {
-                if ((!isCoreQuiet || !isNode())
-                    || debugRejections
-                ) {
+                if (true) {
                     let timeToReject = now - changed[0].time.time;
                     let redMessage = `!!! (VALUE REJECTED) LOCK CONTENTION FIXED VIA REJECTION OF VALUE!!!(rejected after ${timeToReject}ms)`;
                     for (let pathValue of changed) {