querysub 0.428.0 → 0.430.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.428.0",
+  "version": "0.430.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -78,6 +78,8 @@ interface DataSettings {
         undefined
         //  Split strings into multiple buffers
         | 1
+        //  Add transactionPaths support
+        | 2
     );
 }
 
@@ -92,6 +94,7 @@ interface PathValueStructure {
     source?: string;
     lockCount: number;
     updateCount: number;
+    transactionPaths?: string[];
     // IMPORTANT! All of these values must be explicitly copied to PathValue
     //  (we don't spread it), so if you add a new value here, make sure to
     //  update places that use these values (search for canGCValue in this file).
@@ -264,6 +267,69 @@ class PathValueSerializer {
         return readLocks;
     }
 
+    @measureFnc
+    private transactionPathsWrite(writer: Writer, transactionPaths: (string[] | undefined)[]) {
+        let transactionPathsIndexes = new Map<string[] | undefined, number>();
+        for (let pathArray of transactionPaths) {
+            if (transactionPathsIndexes.has(pathArray)) continue;
+            if (!pathArray || pathArray.length === 0) {
+                transactionPathsIndexes.set(pathArray, -1);
+                continue;
+            }
+            transactionPathsIndexes.set(pathArray, transactionPathsIndexes.size);
+        }
+
+        let uniquePathArrays = Array.from(transactionPathsIndexes.keys());
+        writer.writeFloat64(uniquePathArrays.length);
+        for (let pathArray of uniquePathArrays) {
+            if (!pathArray || pathArray.length === 0) {
+                writer.writeFloat64(0);
+            } else {
+                writer.writeFloat64(pathArray.length);
+                for (let pathHash of pathArray) {
+                    writer.writeString(pathHash);
+                }
+            }
+        }
+
+        let indexes = transactionPaths.map(x => transactionPathsIndexes.get(x)!);
+        writer.writeBuffer(asBuffer(new Int32Array(indexes)));
+    }
+
+    @measureFnc
+    private transactionPathsRead(reader: Reader): (string[] | undefined)[] {
+        let uniquePathArrayCount = reader.readFloat64();
+        let uniquePathArrays: (string[] | undefined)[] = [];
+        for (let i = 0; i < uniquePathArrayCount; i++) {
+            let pathCount = reader.readFloat64();
+            if (pathCount === 0) {
+                uniquePathArrays.push(undefined);
+            } else {
+                let paths: string[] = [];
+                for (let j = 0; j < pathCount; j++) {
+                    paths.push(reader.readString());
+                }
+                uniquePathArrays.push(paths);
+            }
+        }
+
+        let indexesBuf = reader.readBuffer();
+        let indexes: number[] = [];
+        for (let i = 0; i < indexesBuf.length; i += 4) {
+            indexes.push(indexesBuf.readInt32LE(i));
+        }
+        let transactionPaths = Array.from(indexes).map(index => {
+            if (index === -1) return undefined;
+            if (index < 0 || index >= uniquePathArrays.length) {
+                debugbreak(2);
+                debugger;
+                throw new Error(`Invalid transaction paths index ${index}, expected 0 <= index < ${uniquePathArrays.length}`);
+            }
+            return uniquePathArrays[index];
+        });
+        return transactionPaths;
+    }
+
 
     private async valuesWrite(writer: Writer, valuesGroups: PathValue[][]) {
         let totalLength = valuesGroups.reduce((total, x) => total + x.length, 0);
@@ -389,7 +455,7 @@ class PathValueSerializer {
         singleBuffer?: boolean;
         stripSource?: boolean;
     }): Promise<Buffer[]> {
-        const version = 1;
+        const version = 2;
         let settings: DataSettings = {
             valueCount: values.length,
             noLocks: config?.noLocks,
@@ -479,6 +545,7 @@ class PathValueSerializer {
             // TODO: Support breaking locks up
             this.readLocksWrite(writer, values.map(x => x.locks));
         }
+        this.transactionPathsWrite(writer, values.map(x => x.transactionPaths));
         await this.valuesWrite(writer, valueGroups);
 
         if (currentBufferPos > 0) {
@@ -560,6 +627,8 @@ class PathValueSerializer {
         // All values become undefined. If also skipping strings OR if you have large values,
         //  this can make deserialization significantly faster.
         skipValues?: boolean;
+        // Skip deserializing transaction paths.
+        skipTransactionPaths?: boolean;
     }): Promise<PathValue[]> {
         buffers = buffers.slice();
 
@@ -684,6 +753,13 @@ class PathValueSerializer {
         let partialValues = this.pathValuesRead(reader, settings.valueCount);
         let readLocks = settings.noLocks ? Array(settings.valueCount).fill([]) : this.readLocksRead(reader);
 
+        let transactionPaths: (string[] | undefined)[] = [];
+        // Skip transactions if they at all seem like they want to skip anything, as transactions are not very important. And how are you going to use transaction paths if you don't have values or if you don't have strings? 
+        // - Oh, but I think skip values just makes the values lazy. So that's normal. We still want transactions even if the values are lazy. 
+        if (version > 1 && !config?.skipStrings && !config?.skipTransactionPaths) {
+            transactionPaths = this.transactionPathsRead(reader);
+        }
+
         let values: unknown[] = [];
         let valuesAreLazy: boolean[] = [];
         if (!config?.skipValues) {
@@ -710,6 +786,7 @@ class PathValueSerializer {
                 source: partialValue.source,
                 lockCount: partialValue.lockCount,
                 updateCount: partialValue.updateCount,
+                transactionPaths: transactionPaths[i],
             });
         }
 

package/src/0-path-value-core/AuthorityLookup.ts

@@ -18,8 +18,8 @@ import { getAllAuthoritySpec, getEmptyAuthoritySpec } from "./PathRouterServerAu
 
 setImmediate(() => import("../3-path-functions/syncSchema"));
 
-let NETWORK_POLL_INTERVAL = timeInMinute * 5;
-let CALL_TIMEOUT = isPublic() ? timeInSecond * 20 : timeInSecond * 3;
+let NETWORK_POLL_INTERVAL = timeInMinute * 1;
+let CALL_TIMEOUT = timeInSecond * 5;
 
 export type AuthorityEntry = {
     nodeId: string;

package/src/0-path-value-core/PathRouter.ts

@@ -11,6 +11,7 @@ import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
 import { sha256 } from "js-sha256";
 import { rangesOverlap, removeRange } from "../rangeMath";
 import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
+import { getBufferInt } from "../bits";
 
 
 // Cases
@@ -190,7 +191,9 @@ export class PathRouter {
             this.lastKeyRoute.route = override.route;
             return override.route;
         }
-        let hash = fastHash(key);
+        //let hash = fastHash(key);
+        // NOTE: Having an even distribution is more important than hashing quickly, and it makes testing a lot easier.
+        let hash = getBufferInt(Buffer.from(sha256(key), "hex"));
         let route = hash % (1000 * 1000 * 1000) / (1000 * 1000 * 1000);
         this.lastKeyRoute.key = key;
         this.lastKeyRoute.route = route;
@@ -636,8 +639,7 @@ export class PathRouter {
             sort(fullPathMatches, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
             sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
             let missingRanges: { start: number; end: number }[] = [{
-                start: 0,
-                end: 1,
+                ...parentRange
             }];
             let usedParts: {
                 nodeId: string;

package/src/0-path-value-core/PathValueController.ts

@@ -9,7 +9,7 @@ import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSeriali
 import { pathValueCommitter } from "./PathValueCommitter";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
 import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
-import { isDiskAudit } from "../config";
+import { getSlowdown, isDiskAudit } from "../config";
 import { decodeNodeId } from "../-a-auth/certs";
 import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
@@ -20,6 +20,7 @@ import { LZ4 } from "../storage/LZ4";
 import { Time } from "./pathValueCore";
 import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
 import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
+import { delay } from "socket-function/src/batching";
 export { pathValueCommitter };
 
 // ONLY returns non-canGCValues that are valid
@@ -127,6 +128,11 @@ export class PathValueControllerBase {
         let callerId = SocketFunction.getCaller().nodeId;
         let { initialCreation, valueBuffers } = config;
 
+        let slowdown = getSlowdown();
+        if (slowdown) {
+            await delay(slowdown);
+        }
+
         let values: PathValue[] = [];
         if (valueBuffers) {
             values = await pathValueSerializer.deserialize(valueBuffers);

package/src/0-path-value-core/pathValueCore.ts

@@ -1,5 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
-import { lazy } from "socket-function/src/caching";
+import { cacheLimited, lazy } from "socket-function/src/caching";
 import { addEpsilons, minusEpsilon } from "../bits";
 import { logErrors } from "../errors";
 import { appendToPathStr, getParentPathStr, getPathDepth, getPathIndexAssert, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath } from "../path";
@@ -26,6 +26,8 @@ import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
 import { decodeParentFilter, filterChildPathsBase } from "./hackedPackedPathParentFiltering";
 import { isDiskAudit } from "../config";
 import { removeRange } from "../rangeMath";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { sha256 } from "js-sha256";
 
 setImmediate(async () => {
     // Import everything will dynamically import, so the client side can tell that it's required. 
@@ -291,6 +293,10 @@ export type PathValue = {
     /** Used internally by archive management services. */
     updateCount?: number;
 
+    /** This is the hashes of all the paths that were written at the same time, as in that have the same time as this path value.
+        - By using this, you can determine if you've only partially read a transaction by looking at the paths which you have read and comparing them against these path values. And if you see that some of them have a time equal to our time, but some of them have a time before our time, it means that you are missing part of our transaction. This allows the client to wait for the rest of the transaction to be received. */
+    transactionPaths?: string[];
+
     // #endregion
 };
 
@@ -374,6 +380,10 @@ export function getLockHash(lock: ReadLock) {
     return timeHash(lock.startTime) + "_" + timeHash(lock.endTime) + "_" + lock.path;
 }
 
+export const hashPathForTransaction = cacheLimited(1000 * 1000, function hashPathForTransaction(path: string): string {
+    return Buffer.from(sha256(path), "hex").toString("base64").slice(0, 8);
+});
+
 export function debugPathValuePath(pathValue: { time: Time; path: string }): string {
     // Log the raw path, so it can show up in searches
     return `(${debugTime(pathValue.time)}) ${pathValue.path}`;
@@ -582,7 +592,7 @@ class AuthorityPathValueStorage {
         }
         return true;
     }
-    public isSynced(path: string, ignoreParentSync = false) {
+    public isSynced(path: string) {
         if (PathRouter.isSelfAuthority(path)) return true;
 
         if (this.isSyncedCache.has(path)) return true;
@@ -597,24 +607,42 @@ class AuthorityPathValueStorage {
             return true;
         }
 
-        // NOTE: Parent syncs sends child values. This helps with speed, and is used in RemoteWatcher
-        //  to prevent extra sends. HOWEVER, we also have to accept a value is synced if only the
-        //  parent is synced
-        //  - See PathWatcher.watchPath, where when !noInitialTrigger (aka, initialTrigger),
-        //      we get all paths via authorityStorage.getPathsFromParent, and send their values.
-        if (!ignoreParentSync && this.isParentSynced(getParentPathStr(path))) {
-            return true;
+        {
+            // We are always going to prefer our own route. So if we can hash with our own route and match our route range, then it's going to be picked 100% of the time. And so it's always going to be synced. 
+            let ourSpec = authorityLookup.getOurSpec();
+            let ourRoute = PathRouter.getRouteFull({ path, spec: ourSpec });
+            if (ourSpec.routeStart <= ourRoute && ourRoute < ourSpec.routeEnd) {
+                this.isSyncedCache.add(path);
+                return true;
+            }
+        }
+
+        let remoteWatchRoute = remoteWatcher.getRemoteParentWatchRoute(path);
+        if (remoteWatchRoute !== undefined) {
+            let parent = getParentPathStr(path);
+            let ranges = this.parentsSynced.get(parent);
+            // NOTE: We can't cache this because when we stop watching the parent, the remote watch will remove it, which will cause us to be no longer synced. However, it's not going to know to go into our synced cache and remove it. 
+            if (ranges === true) return true;
+            if (ranges) {
+                // NOTE: This code is unfortunately fragile. It depends on the ranges that we're watching on the remote to match up with the synced ranges that we're keeping track of. 
+                //  HOWEVER I think it works fine because if we disconnect from a remote and so we stop watching them, the remote watcher will remove it, and so it will no longer match the route. And then it'll try to find something that matches that previous route exactly. Perhaps redoing all of the remote watchers if to in order to find something that hashes consistently. So I think it should be safe. 
+                for (let range of ranges) {
+                    if (range.start <= remoteWatchRoute && remoteWatchRoute < range.end) {
+                        return true;
+                    }
+                }
+            }
         }
 
         return false;
     }
-    public isParentSynced(path: string) {
-        let originalPath = path;
-        let range = decodeParentFilter(path);
-        path = hack_stripPackedPath(path);
-        if (PathRouter.isLocalPath(path)) return true;
+    public isParentSynced(parentPath: string) {
+        let originalPath = parentPath;
+        let range = decodeParentFilter(parentPath);
+        parentPath = hack_stripPackedPath(parentPath);
+        if (PathRouter.isLocalPath(parentPath)) return true;
 
-        let synced = this.parentsSynced.get(originalPath) || this.parentsSynced.get(path);
+        let synced = this.parentsSynced.get(originalPath) || this.parentsSynced.get(parentPath);
         if (synced === true) return true;
         // See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path. 
         if (synced && range) {
@@ -1109,4 +1137,5 @@ setImmediate(() => {
 export { pathValueArchives };
 
 (globalThis as any).core = module.exports;
+(globalThis as any).authorityStorage = authorityStorage;
 (globalThis as any).SocketFunction = SocketFunction;

package/src/1-path-client/RemoteWatcher.ts

@@ -143,6 +143,19 @@ export class RemoteWatcher {
             return x.start <= route && route < x.end;
         });
     }
+    public getRemoteParentWatchRoute(path: string) {
+        let parentPath = getParentPathStr(path);
+        let watchObj = this.remoteWatchParents2.get(parentPath);
+        if (!watchObj) return undefined;
+        // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value. 
+        for (let range of watchObj.ranges) {
+            let route = PathRouter.getRouteFull({ path, spec: range.authoritySpec });
+            if (range.start <= route && route < range.end) {
+                return route;
+            }
+        }
+        return undefined;
+    }
 
 
     public debugIsWatchingPath(path: string) {
@@ -486,7 +499,6 @@ export class RemoteWatcher {
                             auditLog("Asking to watch parent path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
                         }
                     }
-
                     logErrors(RemoteWatcher.REMOTE_WATCH_FUNCTION(config, authorityId));
                 }
             });

package/src/1-path-client/pathValueClientWatcher.ts

@@ -20,7 +20,7 @@ import { logErrors } from "../errors";
 import { getParentPathStr, getPathFromStr, hack_stripPackedPath } from "../path";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
-import { PathValue, Value, getNextTime, Time, ReadLock, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig } from "../0-path-value-core/pathValueCore";
+import { PathValue, Value, getNextTime, Time, ReadLock, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig, hashPathForTransaction } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { blue, green, red } from "socket-function/src/formatting/logColors";
 import { runInfinitePoll } from "socket-function/src/batching";
@@ -681,6 +681,8 @@ export class ClientWatcher {
             /** Causes us to NOT ACTUALLY write values, but just return what we would write, if !dryRun */
             dryRun?: boolean;
             source?: string;
+            // If we read values in the past, not taking the latest value, we need to be told about this because this changes how we will be creating the transaction. 
+            historyBasedReads?: boolean;
         }
     ): PathValue[] {
         const { values, locks, eventPaths } = config;
@@ -704,6 +706,13 @@ export class ClientWatcher {
             source = debugName;
         }
 
+        // Create transaction paths array - hash all paths and reuse the same array for all PathValues
+        // - Sorting doesn't matter. I guess it just makes it so it's technically going to encode more efficiently... or something?
+        let transactionPaths: string[] | undefined = Array.from(values.keys()).map(hashPathForTransaction).sort();
+        if (config.historyBasedReads) {
+            transactionPaths = undefined;
+        }
+
         let pathValues: PathValue[] = [];
         for (let [path, value] of values) {
             let valueIsEvent = event;
@@ -711,6 +720,8 @@ export class ClientWatcher {
                 valueIsEvent = true;
             }
             let isTransparent = value === undefined || config.forcedUndefinedWrites?.has(path);
+            // IMPORTANT! With the exception of epoch values, this is really the only place that actually creates path values. All path values will be created on this specific line.
+            //  - In the future, we could create PathValues in different places. However, that would require PathValue generation that doesn't use ClientWatcher, which would be complex, and probably not worth it. Manually creating path values. While technically faster, the only cases in which you would need that speed would then overload the path value server and all the watchers, and so they're not really viable. Not to mention the fact that using ClientWatcher and Proxy PathWatcher are very fast anyway and can easily create probably hundreds of thousands of writes a second.
             let pathValue: PathValue = pathValueCtor({
                 path,
                 valid: true,
@@ -722,6 +733,7 @@ export class ClientWatcher {
                 event: valueIsEvent,
                 isTransparent,
                 source,
+                transactionPaths,
             });
             pathValues.push(pathValue);
         }

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -234,6 +234,17 @@ export interface WatcherOptions<Result> {
     /** AKA, isAllSynced is always true. Useful for cases when we want to show partially loaded values. */
     commitAllRuns?: boolean;
 
+
+    /** Path values contain some information about all the paths that were written at once, which we call a transaction. We can use this to detect when we are missing part of a transaction and wait until we receive that part of the transaction. 
+        - If we don't wait, our writes will get rejected, and so the system will still be valid, however, there are a few reasons we want to avoid this:
+            - If we are trying to trigger external data, such as API calls, we want to reduce raise conditions. The values still might be rejected entirely, however, at least there won't be a partial read, whereas partial reads are significantly more common than rejections. 
+            - Rejections can cause view stuttering, where the view goes to a bad state before it goes to a good state. If we just wait for the transaction to complete before trying to commit a value, such as in function runner, we can remove this stutter.
+            - Waiting can be more efficient than committing it and having to reject it, especially if something else depends on our value and then it has to be rejected and rerun. 
+            - Not all writers will automatically rerun on rejections. They should be able to handle rejections. However, it might cause downtime and it might have to put the system into a partial recovery state. It's better to avoid this and just wait for the full transaction. 
+        - Because this is so useful and so cheap to calculate, this flag will be set by default by most commit-style templates. It should be safe to set in watch style templates, However, it's not on by default there as, technically speaking, waiting for incomplete transactions will cause higher latency to render, And by default we prefer to render fast and usually get it right rather than render slow and always render the right thing. 
+     */
+    waitForIncompleteTransactions?: boolean;
+
     // NOTE: The reason there isn't throttle support here is very frequently when you want to throttle one component rendering, it's because you have many components. So you actually want to throttle many components and have them throttle in conjunction with each other, which results in the logic becoming complicated.
     //  - But maybe we should support the single throttle case anyways?
 }
@@ -1176,6 +1187,9 @@ export class PathValueProxyWatcher {
             }
         };
         watcher.hasAnyUnsyncedAccesses = () => {
+            if (watcher.options.waitForIncompleteTransactions) {
+                waitIfReceivedIncompleteTransaction();
+            }
             return (
                 watcher.pendingUnsyncedAccesses.size > 0
                 || watcher.pendingUnsyncedParentAccesses.size > 0
@@ -1474,6 +1488,8 @@ export class PathValueProxyWatcher {
                     }
                     watcher.countSinceLastFullSync++;
                     if (watcher.countSinceLastFullSync > 10) {
+                        require("debugbreak")(2);
+                        debugger;
                         console.warn(`Watcher ${watcher.debugName} has been unsynced for ${watcher.countSinceLastFullSync} times. This is fine, but maybe optimize it. Why is it cascading?`, watcher.lastUnsyncedAccesses, watcher.lastUnsyncedParentAccesses, watcher.options.watchFunction);
                     }
                     if (watcher.countSinceLastFullSync > 500) {
@@ -1552,6 +1568,15 @@ export class PathValueProxyWatcher {
 
             if (watcher.pendingWrites.size > 0) {
                 let locks: ReadLock[] = [];
+                let historyBasedReads = false;
+                for (let time of watcher.pendingAccesses.keys()) {
+                    if (time === undefined) continue;
+                    // If we're reading from a specific time, and it isn't just our write time, then we're reading in the past.
+                    if (compareTime(time, writeTime) !== 0) {
+                        historyBasedReads = true;
+                        break;
+                    }
+                }
 
                 if (!watcher.options.noLocks && !watcher.options.unsafeNoLocks) {
                     for (let [readTime, values] of watcher.pendingAccesses) {
@@ -1651,6 +1676,7 @@ export class PathValueProxyWatcher {
                     dryRun: options.dryRun,
                     forcedUndefinedWrites,
                     source: options.source,
+                    historyBasedReads,
                 });
             }
 
@@ -1966,6 +1992,7 @@ export class PathValueProxyWatcher {
     ): Promise<Result> {
         options = {
             canWrite: true,
+            waitForIncompleteTransactions: true,
             ...options,
         };
 
@@ -2507,4 +2534,5 @@ registerPeriodic(function logWatcherTypes() {
 // #endregion Proxy Ordering
 
 import { inlineNestedCalls } from "../3-path-functions/syncSchema"; import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/PathRouter";
+import { waitIfReceivedIncompleteTransaction } from "./TransactionDelayer";
 

package/src/2-proxy/TransactionDelayer.ts

@@ -0,0 +1,95 @@
+import { delay } from "socket-function/src/batching";
+import { PathValue, hashPathForTransaction, compareTime, Time } from "../0-path-value-core/pathValueCore";
+import { proxyWatcher } from "./PathValueProxyWatcher";
+
+const MISSING_TRANSACTION_PART_TIMEOUT = 5000;
+
+export function isMissingTransactionPart(pathValues: PathValue[]): { time: number; path: string; } | undefined {
+    // Create a map of path hash -> time for quick lookups
+    const pathHashToTime = new Map<string, {
+        time: Time;
+        path: string;
+    }>();
+    for (const pv of pathValues) {
+        const hash = hashPathForTransaction(pv.path);
+        const existing = pathHashToTime.get(hash);
+        // Keep the latest time for this path
+        if (!existing || compareTime(pv.time, existing.time) > 0) {
+            pathHashToTime.set(hash, {
+                time: pv.time,
+                path: pv.path,
+            });
+        }
+    }
+
+    // Group PathValues by their transactionPaths array reference
+    const uniqueTransactions = new Map<string[], Time>();
+    for (const pathValue of pathValues) {
+        if (!pathValue.transactionPaths || pathValue.transactionPaths.length === 0) {
+            continue;
+        }
+
+        const existing = uniqueTransactions.get(pathValue.transactionPaths);
+        if (!existing) {
+            uniqueTransactions.set(pathValue.transactionPaths, pathValue.time);
+        }
+    }
+
+    let newestWaitTime: {
+        time: number;
+        path: string;
+    } | undefined = undefined;
+
+    // Check each unique transaction only once
+    for (const [transactionPaths, transactionTime] of uniqueTransactions) {
+        for (const transactionPathHash of transactionPaths) {
+            const pathTime = pathHashToTime.get(transactionPathHash);
+            if (!pathTime) continue;
+
+            // We know that there's a value with this path hash at the transaction time. And so if the value that we've read is before that, then we haven't read the latest value. So we know that we're missing part of the transaction.
+            const timeComparison = compareTime(pathTime.time, transactionTime);
+            if (timeComparison < 0) {
+                if (!newestWaitTime || transactionTime.time > newestWaitTime.time) {
+                    newestWaitTime = {
+                        time: transactionTime.time,
+                        path: pathTime.path,
+                    };
+                }
+            }
+        }
+    }
+    if (!newestWaitTime) return undefined;
+    return newestWaitTime;
+}
+
+export function getAllPendingPathValueReads() {
+    let values: PathValue[] = [];
+    let watcher = proxyWatcher.getTriggeredWatcher();
+    for (let map of watcher.pendingAccesses.values()) {
+        for (let pathValue of map.values()) {
+            values.push(pathValue.pathValue);
+        }
+    }
+    return values;
+}
+
+
+export function waitIfReceivedIncompleteTransaction() {
+    // 1) Keep track of the last proxy watcher we just registered it for. And then if the proxy watcher is the same as that one and that proxy watcher is waiting on a promise, don't register it again. 
+    let watcher = proxyWatcher.getTriggeredWatcher();
+    // We don't want to register a whole bunch of duplicate promises. So if somebody's waiting, there's no need to even do our check. 
+    if (watcher.specialPromiseUnsynced) return;
+
+    let newestTime = isMissingTransactionPart(getAllPendingPathValueReads());
+    if (!newestTime) return;
+
+    let timeout = newestTime.time + MISSING_TRANSACTION_PART_TIMEOUT;
+    let now = Date.now();
+    if (timeout < now) return;
+
+    let promise = delay(timeout - now);
+    // This really nicely both blocks it from finishing because of the waiting for the promise and also triggers it automatically when the delay finishes. HOWEVER, In practice, the promise should never be required to trigger it. It should trigger when we receive the missing parts of the transaction, which we will be, of course, watching as they're in the path values that we're accessing. The timeout is just in case something goes wrong and we incorrectly think we should receive the values, but we won't. That way, eventually, we do commit the value. 
+    // ALSO! Plus the hash I think only stores like 48 bits. So the chance of collision is somewhat high, especially if we're accessing thousands of paths. So sometimes this will trigger even though we're not missing any part just because we had a collision between the path hashes. 
+    proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: "Missing transaction part" });
+    console.error(`(NOT an error, convert this to a warning after we finish testing). Waiting for missing transaction part ${newestTime.path} which was written at time ${newestTime.time} (now is ${now}). We have parts of this transaction, but we are missing this specific path.`);
+}

package/src/4-querysub/querysubPrediction.ts

@@ -131,6 +131,7 @@ function predictCallBase(config: {
         let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
         if (!result) return;
         if (result.lastInternalLoopCount === -1) return;
+        cleanupPrediction();
         actualValueFinished.resolve();
         for (let callback of onPredictionFinishedCallbacks) {
             callback({ callId: call.CallId, result, functionId: call.FunctionId });
@@ -306,7 +307,6 @@ function predictCallBase(config: {
 
     let didCancel = false;
     function rejectPrediction() {
-        predictions;
         if (didCancel) return;
         didCancel = true;
         if (!predictions) return;

package/src/config.ts

@@ -30,6 +30,7 @@ let yargObj = parseArgsFactory()
         desc: "Track all audit logs to disk. This might end up writing A LOT of data."
     })
     .option("logbackblaze", { type: "boolean", desc: "Log all backblaze activity to disk." })
+    .option("slowdown", { type: "number", desc: "Delay all input data values by this amount of time, pretending like we didn't even receive it until this time is up." })
     .argv
     ;
 type QuerysubConfig = {
@@ -51,6 +52,10 @@ let querysubConfig = lazy((): QuerysubConfig => {
     }
 });
 
+export function getSlowdown() {
+    return yargObj.slowdown;
+}
+
 export function isLogBackblaze() {
     return !!yargObj.logbackblaze;
 }

package/src/diagnostics/SyncTestPage.tsx

@@ -12,15 +12,19 @@ import { authorityStorage } from "../0-path-value-core/pathValueCore";
 import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
 import { MachineThreadInfoController } from "./MachineThreadInfo";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { sort } from "socket-function/src/misc";
+import { list, sort } from "socket-function/src/misc";
 import { PathAuditerController } from "./pathAuditer";
 import { t } from "../2-proxy/schema2";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { getAllPendingPathValueReads, isMissingTransactionPart, waitIfReceivedIncompleteTransaction } from "../2-proxy/TransactionDelayer";
+
+module.hotreload = true;
 
-//todonext
-// So... We need to double write, to write to multiple keys at once, in something that's not a lookup, or no, is a lookup, but before we deploy, so it doesn't have the proper or no, a nested lookup, even better. 
 
 let { data, functions } = Querysub.createSchema({
-    values: t.lookup(t.number)
+    values: t.lookup(t.number),
+    cardinality: t.number,
 })({
     functions: {
         setValue(key: string, value: number) {
@@ -29,6 +33,37 @@ let { data, functions } = Querysub.createSchema({
         incrementValue(key: string) {
             data().values[key]++;
         },
+        deleteValue(key: string) {
+            delete data().values[key];
+        },
+        syncValues(keys: string[], removeKeys: string[]) {
+            Querysub.noLocks(() => {
+                for (let key of removeKeys) {
+                    delete data().values[key];
+                }
+                let max = Math.max(...keys.map(x => data().values[x])) + 1;
+                for (let key of keys) {
+                    data().values[key] = max;
+                }
+            });
+        },
+        checkCardinality(keys: string[]) {
+            Querysub.noLocks(() => {
+                let cardinality = new Set(keys.map(x => data().values[x]));
+                let max = Math.max(...keys.map(x => data().values[x]));
+                data().cardinality = cardinality.size + max / 1000;
+                let watcher = proxyWatcher.getTriggeredWatcher();
+                if (watcher.pendingUnsyncedAccesses.size > 0) {
+                    console.log("Unsynced accesses", [...watcher.pendingUnsyncedAccesses]);
+                }
+                for (let path of watcher.pendingWatches.paths) {
+                    if (!path.includes("syncText")) continue;
+                    let pathValue = authorityStorage.getValueAtTime(path, undefined, false, "noAudit");
+                    let source = remoteWatcher.getExistingWatchRemoteNodeId(path);
+                    console.log("Path value", pathValue?.time.time, pathValueSerializer.getPathValue(pathValue), path, source);
+                }
+            });
+        },
     },
     module,
     moduleId: "syncText",
@@ -45,6 +80,26 @@ let { data, functions } = Querysub.createSchema({
                 getKey: (key: string) => key,
             },
         },
+        deleteValue: {
+            keyOverride: {
+                getPrefix: data => data().values,
+                getKey: (key: string) => key,
+            },
+        },
+        syncValues: {
+            nopredict: true,
+            keyOverride: {
+                getPrefix: data => data().values,
+                getKey: () => "f1.2",
+            },
+        },
+        checkCardinality: {
+            nopredict: true,
+            keyOverride: {
+                getPrefix: data => data().values,
+                getKey: () => "f2",
+            },
+        },
     },
     permissions: {
         PERMISSIONS: isSuperUserPERMISSIONS
@@ -98,6 +153,8 @@ export class SyncTestPage extends qreact.Component {
             });
         }
 
+        let keys = Object.keys(values);
+
         return (
             <div className={css.pad2(20).vbox(40)}>
                 <h1>Sync Test Page</h1>
@@ -138,6 +195,7 @@ export class SyncTestPage extends qreact.Component {
                                         functions.setValue(key, +value);
                                         refreshDelayed();
                                     }} />
+                                    <Button onClick={() => { functions.deleteValue(key); refreshDelayed(); }}>Delete</Button>
                                 </div>
                                 <div className={css.hbox(10, 2).wrap.marginLeft(20)}>
                                     {allThreads.map(thread => {
@@ -184,11 +242,49 @@ export class SyncTestPage extends qreact.Component {
                         refreshDelayed();
                     }}
                 />
-                <Button onClick={() => {
-                    syncTestController.refreshAll();
-                }}>
-                    Refresh
-                </Button>
+                <div>
+                    Cardinality: {data().cardinality}
+                </div>
+                <div className={css.hbox(10)}>
+                    <Button onClick={() => {
+                        syncTestController.refreshAll();
+                    }}>
+                        Refresh
+                    </Button>
+
+                    <Button onClick={() => {
+                        let randKeys = list(4).map(x => Math.random() + "");
+                        functions.syncValues(randKeys, keys);
+                        refreshDelayed();
+                    }}>
+                        Set All Values
+                    </Button>
+                    <Button onClick={() => {
+                        functions.syncValues(keys, []);
+                        refreshDelayed();
+                    }}>
+                        Inc All Values
+                    </Button>
+                    <Button onClick={() => {
+                        functions.checkCardinality(keys);
+                    }}>
+                        Check Cardinality
+                    </Button>
+                    <Button onClick={() => {
+                        functions.syncValues(keys, []);
+                        setTimeout(() => {
+                            Querysub.commit(() => {
+                                functions.checkCardinality(keys);
+                            });
+                        }, 500);
+                        refreshDelayed();
+                    }}>
+                        Both
+                    </Button>
+                </div>
+                <div>
+                    {isMissingTransactionPart(getAllPendingPathValueReads()) && "Missing transaction part" || "No missing transaction part"}
+                </div>
             </div>
         );
     }