querysub 0.461.0 → 0.463.0

package/src/0-path-value-core/pathValueCore.ts

@@ -11,7 +11,7 @@ import { delay, runInfinitePoll } from "socket-function/src/batching";
 import { registerMapArrayResource, registerResource } from "../diagnostics/trackResources";
 import { isNode, isNodeTrue, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { AuthoritySpec, LOCAL_DOMAIN_PATH, PathRouter } from "./PathRouter";
 import { formatTime } from "socket-function/src/formatting/format";
 import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { PromiseRace } from "socket-function/src/promiseRace";
@@ -28,6 +28,7 @@ import { isDiskAudit } from "../config";
 import { removeRange } from "../rangeMath";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { sha256 } from "js-sha256";
+import { evaluateValidStates, isServer } from "../config2";
 
 setImmediate(async () => {
     // Import everything will dynamically import, so the client side can tell that it's required. 
@@ -65,6 +66,11 @@ export const MAX_ACCEPTED_CHANGE_AGE = 1000 * 30;
  */
 export const MAX_CHANGE_AGE = MAX_ACCEPTED_CHANGE_AGE * 2;
 
+/** After this we start considering reading values and GCing them. Before this we
+ *      just merge the values, not really processing them.
+ */
+export const VALUE_GC_THRESHOLD = MAX_CHANGE_AGE * 2;
+
 /** Extra time we keep clientside prediction rejections for, to give us time to receive the actual values. */
 export const CLIENTSIDE_PREDICT_LEEWAY = 500;
 
@@ -76,10 +82,7 @@ export const CLIENTSIDE_PREDICT_LEEWAY = 500;
  */
 export const ARCHIVE_FLUSH_LIMIT = Math.max(MAX_CHANGE_AGE * 10, timeInMinute * 60);
 
-/** After this we start considering reading values and GCing them. Before this we
- *      just merge the values, not really processing them.
- */
-export const VALUE_GC_THRESHOLD = ARCHIVE_FLUSH_LIMIT * 3;
+export const ARCHIVE_STABLE_THRESHOLD = ARCHIVE_FLUSH_LIMIT * 2;
 
 // Only start using ARCHIVE_FLUSH_LIMIT after this time.
 //  Before this date, we moves some files around, so their timestamps are too old.
@@ -96,6 +99,12 @@ export const MAX_ARCHIVE_PROPAGATION_DELAY = 1000 * 10;
 /** The time we wait until we cleanup undefined values from memory. */
 export const UNDEFINED_MEMORY_CLEANUP_DELAY = MAX_CHANGE_AGE * 2;
 
+/** If the rejection is that we're missing a value, We defer it for this window.  */
+export const DEFER_LOCK_WINDOW = 2500;
+
+/** Hold predictions a little bit longer so we can wait for multiple parts to come in. We could be smarter about this, but there's not really any downside in waiting a few seconds with the prediction. Predictions really shouldn't be wrong. And if they are, this actually makes it easier to diagnose when they are wrong and then fix the code so they're not wrong.  */
+export const PREDICT_HOLD_TIME = 2500;
+
 export const ARCHIVE_LOOP_DELAY = MAX_CHANGE_AGE * 2;
 
 export const MIN_WAIT_TIME_UNTIL_DISK_FLUSH = MAX_CHANGE_AGE * 2;
@@ -129,6 +138,8 @@ export const STARTUP_CUTOFF_TIME = THREAD_START_TIME + timeInSecond * 30;
 */
 export const MAX_ACCEPTED_AUTHORITY_STARTUP_TIME = timeInMinute * 30;
 
+export const MISSING_TRANSACTION_PART_TIMEOUT = 15000;
+
 export const predictionLockVersion = -2;
 export function isOurPrediction(value: PathValue) {
     return value.locks.length === 1 && value.locks[0].startTime.version === predictionLockVersion;
@@ -167,12 +178,6 @@ export function timeMinusEpsilon(time: Time) {
     return { time: time.time, version: time.version, creatorId: minusEpsilon(time.creatorId) };
 }
 
-function validateLockCount(value: PathValue) {
-    if (value.lockCount === 0 && value.locks.length > 0) {
-        throw new Error(`lockCount is 0, but locks are present (there are ${value.locks.length}), the value is invalid, lockCount MUST be set correctly`);
-    }
-}
-
 // The startTime MUST have a valid value. For `startTime < other.time < endTime`
 //  them there CANNOT be a valid value (if there is, it means there was contention,
 //  and the lock is rejected, invalidating the write that uses it).
@@ -189,11 +194,10 @@ function validateLockCount(value: PathValue) {
 export type ReadLock = {
     path: string;
     // Exclusive (for ranges), often === the time of the value read
-    // === (for single values), often === the time of the value read, but can be any value,
-    //  as long as it exists. UNLESS readIsTransparent, then it it must either not exist,
-    //  or be a transparent write.
+    // === (for single values), often === the time of the value read, but can be any value (although the path is ReadLock.path), as long as it exists. UNLESS readIsTransparent, then this is only used for the range check.
     startTime: Time;
     // Exclusive (for ranges), often === the time we are writing at
+    // SHOULD uniquely identify the ReadLock array. If for some reason, you want to have different read lock arrays, you're going to need to have a different end time. It's probably best to never have different read lock arrays, and you could perhaps have certain values not participate in the locks, that's fine, but you shouldn't have them participate in different locks. 
     endTime: Time;
     // We don't store the ENTIRE value because that would result in sending the entire
     //  program state per write (although I guess if you only read numbers ReadLock
@@ -243,8 +247,7 @@ export type PathValue = {
      */
     locks: ReadLock[];
 
-    /** Should be set EVERY time we have locks. IF lockCount === 0, then we know there are no locks
-     *      (this is preserved through serialization, even when locks aren't).
+    /** Should be set EVERY time we have locks. IF lockCount === 0, then we know there are no locks.
      */
     lockCount: number;
 
@@ -294,7 +297,9 @@ export type PathValue = {
     updateCount?: number;
 
     /** This is the hashes of all the paths that were written at the same time, as in that have the same time as this path value.
-        - By using this, you can determine if you've only partially read a transaction by looking at the paths which you have read and comparing them against these path values. And if you see that some of them have a time equal to our time, but some of them have a time before our time, it means that you are missing part of our transaction. This allows the client to wait for the rest of the transaction to be received. */
+        - By using this, you can determine if you've only partially read a transaction by looking at the paths which you have read and comparing them against these path values. And if you see that some of them have a time equal to our time, but some of them have a time before our time, it means that you are missing part of our transaction. This allows the client to wait for the rest of the transaction to be received.
+        IMPORTANT! All of these paths should have === locks. If we ever want locks to vary... make sure to NOT include those in the same transactionPaths.
+    */
     transactionPaths?: string[];
 
     // #endregion
@@ -389,7 +394,7 @@ export function debugPathValuePath(pathValue: { time: Time; path: string }): str
     return `(${debugTime(pathValue.time)}) ${pathValue.path}`;
 }
 export function debugPathValue(pathValue: PathValue, write?: boolean): string {
-    return `${debugPathValuePath(pathValue)} ${write ? "=" : "==="} ${ellipsisMiddle(String(pathValueSerializer.getPathValue(pathValue)), 200)}`;
+    return `${debugPathValuePath(pathValue)} ${write ? "=" : "==="} ${ellipsisMiddle(String(pathValueSerializer.getPathValue(pathValue)), 200)}${!pathValue.valid && " (invalid)" || ""}`;
 }
 export function debugTime(time: Time): string {
     return `${time.time}[${time.version}]@${time.creatorId.toString().slice(4, 12)}`;
@@ -430,6 +435,8 @@ class AuthorityPathValueStorage {
     >());
 
     public DEBUG_UNWATCH = false;
+    /** Causes us to leak a lot, but useful for debugging */
+    public DEBUG_NEVER_GC = false;
 
     public overrides = new Set<Map<string, PathValue>>();
 
@@ -451,10 +458,10 @@ class AuthorityPathValueStorage {
     //      with time > Date.now(), in the case that our clock is behind the server.
 
     public getValue(path: string): unknown {
-        return pathValueSerializer.getPathValue(this.getValueAtTime(path));
+        return pathValueSerializer.getPathValue(this.getValueAtOrBeforeTime(path));
     }
-    public getValueExact(path: string, time: Time): PathValue | undefined {
-        let value = this.getValueAtTime(path, time, true);
+    public getValueExactMaybeRejected(path: string, time: Time): PathValue | undefined {
+        let value = this.getValueAtOrBeforeTime(path, time, true);
         if (value && compareTime(value.time, time) === 0) {
             return value;
         }
@@ -462,7 +469,7 @@ class AuthorityPathValueStorage {
     }
 
     /** time === undefined gets the latest value */
-    public getValueAtTime(
+    public getValueAtOrBeforeTime(
         path: string,
         // Include all values with other.time <= time, returning the one with the highest time
         //      (either that is valid, or, if readInvalid is true, just the highest time).
@@ -504,7 +511,7 @@ class AuthorityPathValueStorage {
         return this.getValueOrEpochIfSynced(path);
     }
     public getValueBeforeTime(path: string, time: Time): PathValue | undefined {
-        return this.getValueAtTime(path, timeMinusEpsilon(time));
+        return this.getValueAtOrBeforeTime(path, timeMinusEpsilon(time));
     }
     public getValueOrEpochIfSynced(path: string): PathValue | undefined {
         if (this.isSynced(path)) {
@@ -558,6 +565,7 @@ class AuthorityPathValueStorage {
      *      NOOPS if we are the authority of the path.
      */
     public destroyPath(path: string) {
+        if (this.DEBUG_NEVER_GC) return;
         // If we are the authority we:
         //  1) Need to maintain the values for other nodes
         //  2) Don't need to worry about the values getting out of date,
@@ -767,42 +775,34 @@ class AuthorityPathValueStorage {
 
             // Store in global lookup
             {
-                let specialGoldenCase = false;
                 let values = this.values.get(value.path);
                 if (!values) {
                     values = [];
                     this.values.set(value.path, values);
                     this.addParentPath(value.path);
                 }
+
+                // IF a local path, we only store the latest.
+                // NOTE: If !evaluateValidStates(), We could probably get away with not storing the history. However, it is sometimes useful to have the history client-side as it can help us show a more accurate UI state when rejections happen (instead of temporarily reverting to show undefined)
+                if (value.path.startsWith(LOCAL_DOMAIN_PATH)) {
+                    values[0] = value;
+                    continue;
+                }
+
                 let insertIndex = values.findIndex(x => compareTime(x.time, value.time) <= 0);
                 if (insertIndex < 0) {
                     values.push(value);
                 } else {
                     let prev = values[insertIndex];
                     if (prev && compareTime(prev.time, value.time) === 0) {
-                        // NOTE: Before, we used to try to just update the valid state. However, it's kind of unsafe to do this as it results in mutating values, which could make our committer miss a valid state change. Assignment should be just as good. 
-                        values[insertIndex] = value;
-                        // Just update the valid state (we likely aren't an authority on the path anyways)
-                        //prev.valid = value.valid;
+                        // IMPORTANT! We HAVE to update the valid state, otherwise our archive values won't have the correct valid state, as they can't check storage, as we evict values from storage!
+                        // @ts-expect-error
+                        values[insertIndex].valid = value.valid;
                     } else {
-                        // Special case values that always have no locks, by just setting the value, and never splicing
-                        if (insertIndex === 0 && value.lockCount === 0 && values.length <= 1) {
-                            validateLockCount(value);
-                            if (!isCoreQuiet && !doNotArchive) {
-                                if (values.length === 1) {
-                                    console.log(`Clobbering ${debugPathValuePath(values[0])} with ${debugPathValuePath(value)}`);
-                                }
-                            }
-                            specialGoldenCase = true;
-                            values[0] = value;
-                        } else {
-                            values.splice(insertIndex, 0, value);
-                        }
+                        values.splice(insertIndex, 0, value);
                     }
                 }
-                if (!specialGoldenCase) {
-                    this.clearRedundantOldValues(values, now, doNotArchive);
-                }
+                this.clearRedundantOldValues(values, now, doNotArchive);
             }
         }
 
@@ -898,34 +898,19 @@ class AuthorityPathValueStorage {
     //  we will call this anyways).
     // NOTE: values are sorted sorted by -time (so newest are first)
     public clearRedundantOldValues(values: PathValue[], now: number, doNotArchive?: boolean) {
-        // NOTE: We can also delete golden rejections BEFORE the valid golden value. But...
-        //  that is less important, as there are likely to be few rejected values, so this
-        //  is really just designed for removing older golden values (which are probably valid).
+        if (authorityStorage.DEBUG_NEVER_GC) return;
+
+        // IMPORTANT! We actually can't clear as many values as I thought we could because a value might be used in a lock, even though itself it has no locks. 
 
-        // NOTE: We clear values as soon as we can to make FunctionRunners run faster, as any database
-        //  with many writes will almost certainly have many FunctionRunner calls, which all need to synchronize
-        //  as much history as we have. So... the less history, we faster FunctionRunners can sync. AND syncing
-        //  data is the true bottleneck, as it requires network traffic, vs this, which is just manipulating
-        //  some values in memory...
+        // NOTE: We do want to clear as many values as possible because on initial sync we have to send the full history.
+        //      AND, We do an initial sync every time there's a rejection, to EVERY watcher on that path, so... we REALLY want the history to be small... But if it's too small, then we won't be able to do rejections properly. 
 
-        let goldenTime = now - MAX_CHANGE_AGE;
+        let goldenTime = now - VALUE_GC_THRESHOLD;
         // NOTE: Values with no locks are implicitly golden, as they can't be rejected
-        let firstGoldenIndex = values.findIndex(x => x.valid && (x.time.time < goldenTime || x.lockCount === 0));
+        let firstGoldenIndex = values.findIndex(x => x.valid && (x.time.time < goldenTime));
         if (firstGoldenIndex < 0) return;
 
-        // Special case, everything is golden, and the latest value is undefined, and is behind the GC
-        //  point, then delete everything, so constantly changing keys doesn't result in us leaking memory forever.
-        if (
-            firstGoldenIndex === 0
-            && values[0].canGCValue
-            && values[0].time.time < now - VALUE_GC_THRESHOLD
-        ) {
-            if (!isCoreQuiet && !doNotArchive) {
-                console.log(`GCing ALL ${debugPathValuePath(values[0])}`);
-                for (let value of values) {
-                    auditLog("GCing", { path: value.path });
-                }
-            }
+        if (firstGoldenIndex === 0 && values[0].canGCValue) {
             // Mark the value as synced though, as synced means we have received values for this path,
             //  and while the values are undefined and so we are deleting them... we have definitely
             //  received values!
@@ -943,27 +928,19 @@ class AuthorityPathValueStorage {
 
         if (firstGoldenIndex === 0) {
             // Remove everything but the latest value
-            let removed = values.splice(1, values.length - 1);
-            if (!isCoreQuiet && !doNotArchive) {
-                console.log(`GCing everything older than ${debugPathValuePath(values[0])}`);
-                for (let value of removed) {
-                    auditLog("GCing", { path: value.path });
-                }
-            }
+            values.splice(1, values.length - 1);
         } else {
             // Delete everything that is golden, except for the most recent VALID value.
             if (firstGoldenIndex >= 0 && values.length > firstGoldenIndex) {
                 for (let i = values.length - 1; i > firstGoldenIndex; i--) {
-                    let gced = values.pop();
-                    if (!isCoreQuiet && !doNotArchive && gced) {
-                        auditLog("GCing", { path: gced.path });
-                    }
+                    values.pop();
                 }
             }
         }
     }
 
     public removePathFromStorage(path: string, reason: string) {
+        if (authorityStorage.DEBUG_NEVER_GC) return;
         let values = this.values.get(path);
         if (values?.length) {
             this.lastDeleteAt.set(path, values[0].time);
@@ -989,6 +966,7 @@ class AuthorityPathValueStorage {
 
     private possiblyUndefinedPaths = lazy((): { value: Set<string> } => {
         let pathsToCheckPointer = { value: new Set<string>() };
+        if (authorityStorage.DEBUG_NEVER_GC) return pathsToCheckPointer;
 
         runInfinitePoll(UNDEFINED_MEMORY_CLEANUP_DELAY, async () => {
             const { pathWatcher } = await import("./PathWatcher");
@@ -1035,6 +1013,7 @@ class AuthorityPathValueStorage {
 
     private eventPaths = lazy((): { value: Set<string> } => {
         let pathsToCheckPointer = { value: new Set<string>() };
+        if (authorityStorage.DEBUG_NEVER_GC) return pathsToCheckPointer;
 
         let garbageCollectTime = MAX_CHANGE_AGE * 2;
         runInfinitePoll(garbageCollectTime, async () => {

package/src/1-path-client/RemoteWatcher.ts

@@ -21,6 +21,7 @@ import { debugNodeThread } from "../-c-identity/IdentityController";
 import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
 import { decodeParentFilter, encodeParentFilter, registerGetSpecForChildPath } from "../0-path-value-core/hackedPackedPathParentFiltering";
 import { removeRange, rangesOverlap } from "../rangeMath";
+import { evaluateValidStates } from "../config2";
 
 setImmediate(() => import("../4-querysub/Querysub"));
 
@@ -489,7 +490,7 @@ export class RemoteWatcher {
                     let config: WatchConfig = {
                         paths: Array.from(paths),
                         parentPaths: Array.from(parentPaths),
-                        fullHistory: Querysub.SEND_FULL_HISTORY_ON_INITIAL_SYNC,
+                        fullHistory: evaluateValidStates(),
                     };
                     if (isDebugLogEnabled()) {
                         for (let path of paths) {
@@ -691,8 +692,8 @@ registerGetSpecForChildPath(path => {
         if (remoteWatchRange) return remoteWatchRange.authoritySpec;
 
         if (!remoteWatcher.hasAnyDirectPathWatches(path)) {
-            // NOTE: This fall through usually means that we received a value before we started watching it, which is weird. That really shouldn't happen often. It'll probably break things. We will probably ignore this value thinking it has a different hash than it does.
-            console.warn(`We do not own a path, but also aren't watching it on a remote, but... receive the value? This is weird, and will probably break things (we will likely ignore this value / not trigger parent watches for it).`, { path });
+            // NOTE: This might mean that our Path Watcher and remote watcher are out of sync, which is a big problem and will break things. 
+            console.error(`We do not own a path, but also aren't watching it on a remote, but... receive the value? This is weird, and will probably break things (we will likely ignore this value / not trigger parent watches for it).`, { path });
         }
     }
     return authorityLookup.getOurSpec();

package/src/1-path-client/pathValueClientWatcher.ts

@@ -20,7 +20,7 @@ import { logErrors } from "../errors";
 import { getParentPathStr, getPathFromStr, hack_stripPackedPath } from "../path";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
-import { PathValue, Value, getNextTime, Time, ReadLock, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig, hashPathForTransaction } from "../0-path-value-core/pathValueCore";
+import { PathValue, Value, getNextTime, Time, ReadLock, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig, hashPathForTransaction, debugTime, compareTime } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { blue, green, red } from "socket-function/src/formatting/logColors";
 import { runInfinitePoll } from "socket-function/src/batching";
@@ -32,6 +32,7 @@ import { remoteWatcher } from "./RemoteWatcher";
 import { heapTagObj } from "../diagnostics/heapTag";
 import { isDevDebugbreak } from "../config";
 import { matchesParentRangeFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
+import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/PathRouter";
 
 const pathValueCtor = heapTagObj("PathValue");
 
@@ -685,7 +686,25 @@ export class ClientWatcher {
             historyBasedReads?: boolean;
         }
     ): PathValue[] {
-        const { values, locks, eventPaths } = config;
+        let { values, locks, eventPaths } = config;
+
+        locks = locks.filter(x => {
+            if (x.path.startsWith(LOCAL_DOMAIN_PATH)) {
+                console.error(`Tried to lock a local path. This would result in expected rejections, ignoring lock`, { path: x.path });
+                return false;
+            }
+            return true;
+        });
+
+        // Ensure all locks have the same end time. 
+        if (locks.length > 0) {
+            let endTime = locks[0].endTime;
+            for (let lock of locks) {
+                if (compareTime(lock.endTime, endTime) !== 0) {
+                    console.error(`Locks have inconsistent end times. ${debugTime(lock.endTime)} !== ${debugTime(endTime)}. Path: ${lock.path}`);
+                }
+            }
+        }
 
         // We trust our caller to use getFreeWriteTime, as they need to know the exact time
         //  they are using so they can use it for their readTimes
@@ -737,6 +756,13 @@ export class ClientWatcher {
             });
             pathValues.push(pathValue);
         }
+        for (let pathValue of pathValues) {
+            // Ensure local values never have any locks. Local rejections are unexpected anyways, and they will not be rerun by the function runner, so they are generally worse than leaving an inconsistent value.
+            if (pathValue.locks.length > 0 && pathValue.path.startsWith(LOCAL_DOMAIN_PATH)) {
+                pathValue.locks = [];
+                pathValue.lockCount = 0;
+            }
+        }
 
         if (!config.dryRun) {
             for (let pv of pathValues) {

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -13,7 +13,7 @@ import { ActionsHistory } from "../diagnostics/ActionsHistory";
 import { registerResource } from "../diagnostics/trackResources";
 import { ClientWatcher, WatchSpecData, clientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { createPathValueProxy, getPathFromProxy, getProxyPath, getProxyPathAndWatch, isValueProxy, isValueProxy2 } from "./pathValueProxy";
-import { authorityStorage, compareTime, ReadLock, epochTime, getNextTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time, getCreatorId, getTimeUnique } from "../0-path-value-core/pathValueCore";
+import { authorityStorage, compareTime, ReadLock, epochTime, getNextTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time, getCreatorId, getTimeUnique, debugTime } from "../0-path-value-core/pathValueCore";
 import { runCodeWithDatabase, rawSchema } from "./pathDatabaseProxyBase";
 import debugbreak from "debugbreak";
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
@@ -589,7 +589,7 @@ export class PathValueProxyWatcher {
 
         const currentReadTime = watcher.options.forceReadLatest ? undefined : watcher.currentReadTime;
 
-        let pathValue = authorityStorage.getValueAtTime(pathStr, currentReadTime);
+        let pathValue = authorityStorage.getValueAtOrBeforeTime(pathStr, currentReadTime);
         if (!watcher.options.noSyncing) {
             // NOTE: If we have any value, we are always synced (that's what synced means, as if we aren't syncing,
             //  we delete any values, to prevent stale values from being used).
@@ -1289,6 +1289,14 @@ export class PathValueProxyWatcher {
             // The read time equals the write time, so our locks use the same time as our write time, and
             //  therefore lock the actual state we used.
             watcher.currentReadTime = watcher.nextWriteTime;
+            // We need to actually read well before our write, otherwise, we might read previous iterations of the same write. This shouldn't break anything, As I don't think the callers really care about when we read from, as long as it's similar to the write time.
+            if (watcher.currentReadTime) {
+                watcher.currentReadTime = {
+                    time: watcher.currentReadTime.time,
+                    version: watcher.currentReadTime.version - 1_000_000,
+                    creatorId: watcher.currentReadTime.creatorId,
+                };
+            }
 
             watcher.permissionsChecker = options.getPermissionsCheck?.();
 
@@ -1550,13 +1558,16 @@ export class PathValueProxyWatcher {
             let setValues: PathValue[] | undefined;
 
             let prefixes = options.overrideAllowLockDomainsPrefixes || getAllowedLockDomainsPrefixes();
-            // This consumes our write time, as once we write, we can't reuse writeTimes!
-            //  (If we do, it breaks dependencies).
+            let currentReadTime = watcher.currentReadTime;
+            if (!currentReadTime) {
+                currentReadTime = getNextTime();
+            }
+
             let writeTime = watcher.nextWriteTime;
-            watcher.nextWriteTime = undefined;
             if (!writeTime) {
                 writeTime = getNextTime();
             }
+            watcher.nextWriteTime = undefined;
 
             if (
                 !watcher.options.hasSideEffects
@@ -1573,7 +1584,7 @@ export class PathValueProxyWatcher {
                 for (let time of watcher.pendingAccesses.keys()) {
                     if (time === undefined) continue;
                     // If we're reading from a specific time, and it isn't just our write time, then we're reading in the past.
-                    if (compareTime(time, writeTime) !== 0) {
+                    if (compareTime(time, currentReadTime) !== 0) {
                         historyBasedReads = true;
                         break;
                     }
@@ -1582,13 +1593,13 @@ export class PathValueProxyWatcher {
                 if (!watcher.options.noLocks && !watcher.options.unsafeNoLocks) {
                     for (let [readTime, values] of watcher.pendingAccesses) {
                         if (!readTime) {
-                            readTime = writeTime;
+                            readTime = currentReadTime;
                         }
                         for (let valueObj of values.values()) {
                             if (valueObj.noLocks) continue;
                             let value = valueObj.pathValue;
-                            // If any value has no locks, AND is local, it won't be rejected, so we don't need to lock it
-                            if (value.lockCount === 0 && value.path.startsWith(LOCAL_DOMAIN_PATH)) {
+                            // NO LOCKS on local paths. Local rejections are unexpected anyways, and they will not be rerun by the function runner, so they are generally worse than leaving an inconsistent value.
+                            if (value.path.startsWith(LOCAL_DOMAIN_PATH)) {
                                 continue;
                             }
                             if (!prefixes.some(x => value.path.startsWith(x))) continue;
@@ -1608,7 +1619,7 @@ export class PathValueProxyWatcher {
                     }
                     for (let [readTime, paths] of watcher.pendingEpochAccesses) {
                         if (!readTime) {
-                            readTime = writeTime;
+                            readTime = currentReadTime;
                         }
                         for (let path of paths) {
                             if (!prefixes.some(x => path.startsWith(x))) continue;
@@ -1629,23 +1640,17 @@ export class PathValueProxyWatcher {
                     //      run in the present, but intentionally want to write in the past? This usually only
                     //      happens if we intentionally run in the past (in which case writeTime will be set,
                     //      so this if statement wouldn't run anyways).
-                    let fixedWriteTime = writeTime;
                     for (let lock of locks) {
-                        if (compareTime(lock.endTime, fixedWriteTime) > 0) {
-                            fixedWriteTime = {
+                        if (compareTime(lock.endTime, writeTime) > 0) {
+                            // I'm curious how this could even happen?
+                            require("debugbreak")(2);
+                            debugger;
+                            console.error(`Write time is before the end time of a lock. This is not allowed. Shifting write time forward ${debugTime(writeTime)} < ${debugTime(lock.endTime)}`);
+                            writeTime = clientWatcher.getFreeWriteTime({
                                 time: addEpsilons(lock.endTime.time, 1),
-                                version: 0,
+                                version: lock.endTime.version,
                                 creatorId: getCreatorId(),
-                            };
-                        }
-                    }
-                    if (fixedWriteTime !== writeTime) {
-                        let prevWriteTime = writeTime;
-                        writeTime = fixedWriteTime;
-                        for (let lock of locks) {
-                            if (lock.endTime === prevWriteTime) {
-                                lock.endTime = writeTime;
-                            }
+                            });
                         }
                     }
                 }

package/src/2-proxy/TransactionDelayer.ts

@@ -1,23 +1,29 @@
 import { delay } from "socket-function/src/batching";
-import { PathValue, hashPathForTransaction, compareTime, Time } from "../0-path-value-core/pathValueCore";
+import { PathValue, hashPathForTransaction, compareTime, Time, authorityStorage, epochTime, createMissingEpochValue, MISSING_TRANSACTION_PART_TIMEOUT, debugPathValue } from "../0-path-value-core/pathValueCore";
 import { SyncWatcher, proxyWatcher } from "./PathValueProxyWatcher";
+import { isDefined } from "../misc";
 
-const MISSING_TRANSACTION_PART_TIMEOUT = 5000;
-
-export function isMissingTransactionPart(pathValues: PathValue[]): { time: number; path: string; } | undefined {
+// NOTE: If we haven't synchronized a path, we'll know it's not synchronized. However, we might have synchronized the path, but we might have an older value. If we take one of the values we've read, we find all the transaction paths in it (which it will tell us directly), And there's other paths that we've also read at the same time in our path values. Except they have a time before the write of the value that we looked at the transaction paths, then it means we don't have the latest values for that transaction. And because we have the path value, we know we're watching it because we watch everything that we access. And so we know we're missing part of the transaction and should wait. 
+export function isMissingTransactionPart(pathValues: PathValue[], timeoutCheck?: boolean): { time: number; path: string; } | undefined {
     // Create a map of path hash -> time for quick lookups
     const pathHashToTime = new Map<string, {
         time: Time;
         path: string;
     }>();
-    for (const pv of pathValues) {
-        const hash = hashPathForTransaction(pv.path);
+    for (const pathValue of pathValues) {
+        if (!pathValue.valid) {
+            if (timeoutCheck) return undefined;
+            require("debugbreak")(2);
+            debugger;
+            throw new Error(`Cannot validate a transaction where part of it is invalid. We shouldn't have even been passed this value. Value: ${debugPathValue(pathValue)}`);
+        }
+        const hash = hashPathForTransaction(pathValue.path);
         const existing = pathHashToTime.get(hash);
         // Keep the latest time for this path
-        if (!existing || compareTime(pv.time, existing.time) > 0) {
+        if (!existing || compareTime(pathValue.time, existing.time) > 0) {
             pathHashToTime.set(hash, {
-                time: pv.time,
-                path: pv.path,
+                time: pathValue.time,
+                path: pathValue.path,
             });
         }
     }
@@ -28,9 +34,7 @@ export function isMissingTransactionPart(pathValues: PathValue[]): { time: numbe
         if (!pathValue.transactionPaths || pathValue.transactionPaths.length === 0) {
             continue;
         }
-
-        const existing = uniqueTransactions.get(pathValue.transactionPaths);
-        if (!existing) {
+        if (!uniqueTransactions.has(pathValue.transactionPaths)) {
             uniqueTransactions.set(pathValue.transactionPaths, pathValue.time);
         }
     }
@@ -41,18 +45,18 @@ export function isMissingTransactionPart(pathValues: PathValue[]): { time: numbe
     } | undefined = undefined;
 
     // Check each unique transaction only once
-    for (const [transactionPaths, transactionTime] of uniqueTransactions) {
-        for (const transactionPathHash of transactionPaths) {
-            const pathTime = pathHashToTime.get(transactionPathHash);
-            if (!pathTime) continue;
-
-            // We know that there's a value with this path hash at the transaction time. And so if the value that we've read is before that, then we haven't read the latest value. So we know that we're missing part of the transaction.
-            const timeComparison = compareTime(pathTime.time, transactionTime);
-            if (timeComparison < 0) {
+    for (let [transactionPaths, transactionTime] of uniqueTransactions) {
+        // It's normal if some of the paths aren't used. That's fine. You're not going to use the entire part of the transaction. Just because someone wrote a whole bunch of values at once doesn't mean you're gonna to read them all at once. 
+        let values = transactionPaths.map(path => pathHashToTime.get(hashPathForTransaction(path))).filter(isDefined);
+        for (let value of values) {
+            // We KNOW there's a value for all the transactions paths at transactionTime. We might have a NEWER value as well, so we don't do !==, BUT, if we only have an older value, it means we need to wait to receive the full transaction. 
+            if (compareTime(value.time, transactionTime) < 0) {
+                // Keep track of the newest transaction time, not the value time. The transaction time is the thing we're waiting for. We're waiting for it to get old enough that we should have received the value. And if we don't receive it by then, we can warn. 
+                //  - If it's an older value, then we would have received it, unless we weren't watching the path. And if we just watch the path, then our synchronization check will ensure that we receive all of the paths. So this is only needed for newer values on paths that we're already watching, in which case, the timer receive it should be very similar to the transaction time. 
                 if (!newestWaitTime || transactionTime.time > newestWaitTime.time) {
                     newestWaitTime = {
                         time: transactionTime.time,
-                        path: pathTime.path,
+                        path: transactionPaths[0],
                     };
                 }
             }
@@ -69,6 +73,16 @@ export function getAllPendingPathValueReads(watcher = proxyWatcher.getTriggeredW
             values.push(pathValue.pathValue);
         }
     }
+    // NOTE: These can't contribute the transaction paths, but they do contribute their path and their time, which will cause other transaction paths to know that they are invalid. 
+    let allEpochPaths = new Set<string>();
+    for (let paths of watcher.pendingEpochAccesses.values()) {
+        for (let path of paths) {
+            allEpochPaths.add(path);
+        }
+    }
+    for (let path of allEpochPaths) {
+        values.push(createMissingEpochValue(path));
+    }
     return values;
 }
 
@@ -77,7 +91,9 @@ export function waitIfReceivedIncompleteTransaction(watcher: SyncWatcher) {
     // We don't want to register a whole bunch of duplicate promises. So if somebody's waiting, there's no need to even do our check. 
     if (watcher.specialPromiseUnsynced) return;
 
-    let newestTime = isMissingTransactionPart(getAllPendingPathValueReads(watcher));
+    let readTime = watcher.currentReadTime;
+    let pendingValuePaths = getAllPendingPathValueReads(watcher);
+    const newestTime = isMissingTransactionPart(pendingValuePaths);
     if (!newestTime) return;
 
     let timeout = newestTime.time + MISSING_TRANSACTION_PART_TIMEOUT;
@@ -85,6 +101,12 @@ export function waitIfReceivedIncompleteTransaction(watcher: SyncWatcher) {
     if (timeout < now) return;
 
     let promise = delay(timeout - now);
+    void promise.finally(() => {
+        let latestValues = pendingValuePaths.map(x => authorityStorage.getValueAtOrBeforeTime(x.path, readTime) || createMissingEpochValue(x.path));
+        if (isMissingTransactionPart(latestValues, true)) {
+            console.error(`Transaction never became complete, at path ${newestTime.path} for time ${newestTime.time}`, { readTime, paths: pendingValuePaths.map(x => x.path) });
+        }
+    });
     // This really nicely both blocks it from finishing because of the waiting for the promise and also triggers it automatically when the delay finishes. HOWEVER, In practice, the promise should never be required to trigger it. It should trigger when we receive the missing parts of the transaction, which we will be, of course, watching as they're in the path values that we're accessing. The timeout is just in case something goes wrong and we incorrectly think we should receive the values, but we won't. That way, eventually, we do commit the value. 
     // ALSO! Plus the hash I think only stores like 48 bits. So the chance of collision is somewhat high, especially if we're accessing thousands of paths. So sometimes this will trigger even though we're not missing any part just because we had a collision between the path hashes. 
     proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: "Missing transaction part" });