querysub 0.403.0 → 0.404.0

package/src/0-path-value-core/PathValueCommitter.ts

@@ -1,41 +1,42 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, batchFunction } from "socket-function/src/batching";
-import { markArrayAsSplitable } from "socket-function/src/fixLargeNetworkCalls";
 import { isNode, timeInSecond } from "socket-function/src/misc";
-import { measureBlock, measureFnc, registerMeasureInfo } from "socket-function/src/profiling/measure";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
-import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
 import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
 import { getPathFromStr } from "../path";
-import { nodePathAuthority, pathValueAuthority2 } from "./NodePathAuthorities";
-import { PathValueController } from "./PathValueController";
-import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath, compareTime, epochTime, isOurPrediction, timeHash } from "./pathValueCore";
+import { PathValueControllerBase } from "./PathValueController";
+import { PathRouter } from "./PathRouter";
+import { PathValue, MAX_ACCEPTED_CHANGE_AGE, WriteState, debugPathValuePath, compareTime, epochTime } from "./pathValueCore";
+import { validStateComputer } from "./ValidStateComputer";
 import debugbreak from "debugbreak";
 import { red } from "socket-function/src/formatting/logColors";
-import { registerDynamicResource } from "../diagnostics/trackResources";
-import { StatsValue, addToStats, addToStatsValue, createStatsValue, getStatsTop } from "socket-function/src/profiling/stats";
-import { formatStats } from "socket-function/src/profiling/statsFormat";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { isClient } from "../config2";
-import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { logDisk } from "../diagnostics/logs/diskLogger";
-
-/*
-- commitWrites <= creating writes
-    - ingestValues <= forwardWrites
-        - ingestValidStates <= forwarded valid states
-            = loop
-                - setValidState
-                - gather PathValue changes
-                - getValidStateChangedTriggers
-                - calculateInitialValidState
-            - REMOTE.ingestValidStates => forwarded valid states
-            - pathWatcher.triggerValuesChanged => forwardWrites + local clients
-    - broadcastValues
-*/
+import { authorityLookup } from "./AuthorityLookup";
+setImmediate(() => import("../1-path-client/RemoteWatcher"));
+setImmediate(() => import("../4-querysub/Querysub"));
+
+const MAX_SEND_TRY_COUNT = 3;
+
+export type BatchValues = {
+    pathValues: PathValue[],
+    parentsSynced?: string[];
+    sourceNodeId: string;
+    initialTrigger?: "initialTrigger";
+    authoritySync?: boolean;
+};
+
+export type RemoteValueAndValidState = {
+    sourceNodeId: string;
+    pathValues: PathValue[];
+    validStates: WriteState[];
+    initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+    // Means it's from an authority path sync
+    authoritySyncPaths?: Set<string>;
+};
 
 class PathValueCommitter {
     private pendingCommits = new Set<Promise<unknown>>();
@@ -95,7 +96,7 @@ class PathValueCommitter {
         if (predictWrites) {
             pathValuesToIngest = values;
         } else {
-            pathValuesToIngest = values.filter(pathValue => pathValueAuthority2.isSelfAuthority(pathValue.path));
+            pathValuesToIngest = values.filter(pathValue => PathRouter.isSelfAuthority(pathValue.path));
         }
 
         // NOTE: Error out early on the client, so we get better errors AND so we can avoid even ingesting the values
@@ -107,54 +108,110 @@ class PathValueCommitter {
         //          (It could even populate on the first domain, so it would always populate on the first write
         //          on the current domain).
         if (!isNode()) {
-            let remoteWrites = values.filter(pathValue => !pathValueAuthority2.isSelfAuthority(pathValue.path));
+            let remoteWrites = values.filter(pathValue => !PathRouter.isSelfAuthority(pathValue.path));
             if (remoteWrites.length > 0) {
                 throw new Error(`Cannot commit writes to paths that are not local to this node. Paths: ${remoteWrites.map(x => x.path).join(", ")}`);
             }
         }
 
         // NOTE: This function will just ignore writes we aren't an authority on, so we can just give it everything.
-        this.ingestValues(pathValuesToIngest, undefined, undefined);
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: pathValuesToIngest,
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+        });
+
+        let remoteValues = values.filter(x => !PathRouter.isSelfAuthority(x.path));
 
-        // If all values are local, then there is nothing to broadcast
-        if (values.some(x => !pathValueAuthority2.isLocalPath(x.path))) {
-            this.addCommitPromise(this.broadcastValues(new Set(values)));
+        if (remoteValues.length > 0) {
+            this.addCommitPromise(this.broadcastValues({ values: new Set(remoteValues) }));
+        }
+
+        // If we're the authority on it, we should still share it with the other authorities. 
+        let stillShareValues = values.filter(x => !PathRouter.isLocalPath(x.path) && PathRouter.isSelfAuthority(x.path));
+        if (stillShareValues.length > 0) {
+            this.addCommitPromise(PathValueControllerBase.authorityShareValues({ pathValues: stillShareValues }));
         }
     }
 
     private broadcastValues = batchFunction(
         { delay: 10, throttleWindow: 500, noMeasure: true },
-        async function internal_forwardWrites(valuesBatched: Set<PathValue>[]) {
-            let values = new Set(valuesBatched.flatMap(x => Array.from(x)));
+        async function internal_forwardWrites(valuesBatched: {
+            values: Set<PathValue>;
+            tryCount?: number;
+        }[]) {
+            let values = new Set(valuesBatched.flatMap(x => Array.from(x.values)));
+            let tryCountPerValue = new Map<PathValue, number>();
+            for (let list of valuesBatched) {
+                for (let value of list.values) {
+                    if (!list.tryCount) continue;
+                    tryCountPerValue.set(value, list.tryCount);
+                }
+            }
+            if (tryCountPerValue.size > 0) {
+                console.info(`Syncing all authorities to ensure we have the latest values, due to failed writes.`);
+                await authorityLookup.syncAllNow();
+            }
+
 
             let valuesPerOtherAuthority = new Map<string, PathValue[]>();
             for (let pathValue of values) {
                 if (isDebugLogEnabled()) {
-                    auditLog("CREATE VALUE", { path: pathValue.path, time: pathValue.time.time });
+                    // let valueStr: string | undefined;
+                    // if (typeof pathValue.value === "boolean" || typeof pathValue.value === "number") {
+                    //     valueStr = pathValue.value.toString();
+                    // } else if (typeof pathValue.value === "string") {
+                    //     valueStr = pathValue.value.slice(0, 100);
+                    //     if (valueStr.length < pathValue.value.length) {
+                    //         valueStr += `[+${pathValue.value.length - valueStr.length}]`;
+                    //     }
+                    // } else if (Buffer.isBuffer(pathValue.value)) {
+                    //     valueStr = `Buffer(${pathValue.value.length})`;
+                    // } else if (pathValue.value === null) {
+                    //     valueStr = "{null}";
+                    // } else {
+                    //     valueStr = `{${typeof pathValue.value}}`;
+                    // }
+                    auditLog("CREATE VALUE", {
+                        path: pathValue.path,
+                        timeId: pathValue.time.time,
+                        source: pathValue.source,
+                        transparent: pathValue.isTransparent,
+                        // value: valueStr,
+                        event: pathValue.event,
+                        // NOTE: I think it'd be too expensive to log all the locks. There's one thing to log every time we read a value or create a value, but to log all of the dependencies for each write path would be O(locksPerWrite * 2 * writes), which is just too much, and could easily result in ten thousand logs per write. I think it will be fine because if there's a rejection, then that will tell us the time ID that the rejected value was reading, Which we can infer to know at least one of the locks, as in the most important lock that that right had.
+                    });
                 }
 
-                let otherAuthorities = await pathValueAuthority2.getWriteNodes(pathValue.path);
-                otherAuthorities = otherAuthorities.filter(x => !isOwnNodeId(x));
-                if (otherAuthorities.length === 0 && !pathValueAuthority2.isSelfAuthority(pathValue.path)) {
-                    throw new Error(`There are no authorities for path ${pathValue.path}. The write will be lost.`);
+                let otherAuthorities = PathRouter.getAllAuthorities(pathValue.path);
+                otherAuthorities = otherAuthorities.filter(x => !isOwnNodeId(x.nodeId));
+                if (otherAuthorities.length === 0) {
+                    validStateComputer.ingestValuesAndValidStates({
+                        pathValues: [{
+                            ...pathValue,
+                            valid: false,
+                        }],
+                        parentSyncs: [],
+                        initialTriggers: { values: new Set(), parentPaths: new Set() },
+                    });
+                    PathRouter.getAllAuthorities(pathValue.path);
+                    console.error(`There are no authorities for path ${pathValue.path}. The write will be lost.`, {
+                        path: pathValue.path,
+                        timeId: pathValue.time.time,
+                        source: pathValue.source,
+                        otherAuthorities,
+                    });
                 }
                 for (let otherAuthority of otherAuthorities) {
-                    let values = valuesPerOtherAuthority.get(otherAuthority);
+                    let values = valuesPerOtherAuthority.get(otherAuthority.nodeId);
                     if (!values) {
                         values = [];
-                        valuesPerOtherAuthority.set(otherAuthority, values);
+                        valuesPerOtherAuthority.set(otherAuthority.nodeId, values);
                     }
                     values.push(pathValue);
                 }
             }
 
-            // for (let [authorityId, values] of valuesPerOtherAuthority) {
-            //     console.log(`Sending ${values.length} values to ${authorityId} (${pathValueAuthority2.getAuthorityPaths(authorityId)?.map(x => pathValueAuthority2.getArchiveDirectory(x)).join(" | ")})`);
-            //     // for (let value of values) {
-            //     //     console.log("    " + value.path);
-            //     // }
-            // }
-
             // Don't send to bad nodes for 60 seconds
             const nodeIgnoreTime = Date.now() - 1000 * 60;
             let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
@@ -174,39 +231,58 @@ class PathValueCommitter {
                 //  to be on a trusted node (which should always be a server), so... either our network went
                 //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
                 //  writes are going to too old and therefore rejected by the time a server is up anyways.
-                markArrayAsSplitable(values);
-                const { Querysub } = await import("../4-querysub/Querysub");
-                let serializedValues = await pathValueSerializer.serialize(values, { compress: Querysub.COMPRESS_NETWORK });
-                console.info("Send PathValues to server", { valueCount: values.length, targetId: otherAuthority, });
-                let forwardPromise = PathValueController.nodes[otherAuthority].forwardWrites(
-                    serializedValues,
-                    undefined,
-                    "watchLocks"
-                );
-                pathValueCommitter.addCommitPromise(forwardPromise);
+
+                let forwardPromise = PathValueControllerBase.createValues({
+                    nodeId: otherAuthority,
+                    pathValues: values,
+                });
                 logErrors(forwardPromise);
-                // NOTE: Usually we don't need watch the locks for our values because upon
-                //  forwarding them the server will automatically watch them. However...
-                //  if the call errors out, we can't know if we are watching them, so
-                //  we have to explicitly find a server to tell us if our writes go through
-                //  (we can't even just drop our writes, as we don't differentiate predicted
-                //  and real writes, so we can't tell if it got through and was sent back,
-                //  or if it was never committed!)
-                forwardPromise.catch(() => {
-
-                    // TODO: For the first few seconds, look for new writers and retry. However,
-                    //      after the write is too old, just log the error to the user, as
-                    //      at that point even if a writer appears the write will be too stale.
-
-                    // Watch the readLocks, so we know if the values are valid or not (this
-                    //  code has retry logic).
-                    //  NOTE: Technically if this talks to another authority, and then the original authority
-                    //      comes back, we might be syncing valid states from multiple authorities, and as
-                    //      they aren't timestamped, we could receive an older value later. Which should
-                    //      correct itself, but... we would also be subscribed to multiple authorities,
-                    //      for the same path, which can result in thrashing. But... this probably won't happen...
-                    lockWatcher.watchValueLocks(values);
+                void forwardPromise.then(x => {
+                    if (x === "refused") {
+                        values = values.map(value => {
+                            console.info(`Rejecting past value due to initial sync: ${debugPathValuePath(value)}`, {
+                                path: value.path,
+                                timeId: value.time.time,
+                            });
+                            return {
+                                ...value,
+                                valid: false,
+                            };
+                        });
+
+                        validStateComputer.ingestValuesAndValidStates({
+                            pathValues: values,
+                            parentSyncs: [],
+                            initialTriggers: { values: new Set(), parentPaths: new Set() },
+                        });
+                    }
                 });
+                void forwardPromise.catch(async error => {
+                    let byTryCount = new Map<number, PathValue[]>();
+                    for (let value of values) {
+                        let tryCount = tryCountPerValue.get(value) ?? 0;
+                        let arr = byTryCount.get(tryCount) ?? [];
+                        arr.push(value);
+                        byTryCount.set(tryCount, arr);
+                    }
+                    for (let [tryCount, values] of byTryCount.entries()) {
+                        tryCount++;
+                        if (tryCount > MAX_SEND_TRY_COUNT) {
+                            console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
+                                error: error.message,
+                                otherAuthority,
+                                count: values.length,
+                            });
+                            continue;
+                        }
+                        void pathValueCommitter.broadcastValues({
+                            values: new Set(values),
+                            tryCount: tryCount,
+                        });
+                    }
+                });
+
+                pathValueCommitter.addCommitPromise(forwardPromise);
             });
             // await, so "waitForValuesToCommit" works correctly
             await Promise.all(promises.map(x => timeoutToUndefined(timeInSecond * 30, errorToUndefined(x))));
@@ -214,47 +290,33 @@ class PathValueCommitter {
     );
 
 
-    private getExistingWatchRemoteNodeId = (path: string): string | undefined => undefined;
-    public setGetRemoteWatchNodeId(fnc: (path: string) => string | undefined) {
-        this.getExistingWatchRemoteNodeId = fnc;
-    }
+    public ingestRemoteValuesAndValidStates = batchFunction(
+        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
+        async (batched: RemoteValueAndValidState[]) => {
+            const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
 
-    /** When ingesting remote values we batch a bit. Otherwise a stream of values will often result in WAY too many
-     *      clientside evaluations. The network will already add a lot of delay, so this shouldn't be noticeable.
-     *      - AND, with the throttleWindow, we generally won't even be waiting anyways.
-     *      - The delay is 16ms, as we won't render to the screen faster than that anyways, so handling updates
-     *          at a rate faster than that will just pointlessly add lag.
-     */
-    public ingestRemoteValues = batchFunction(
-        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValues", noMeasure: true },
-        async (batched: {
-            pathValues: PathValue[],
-            parentsSynced?: string[];
-            sourceNodeId: string;
-            initialTrigger?: "initialTrigger";
-        }[]) => {
-            let initialTriggers = { values: new Set<string>(), parentPaths: new Set<string>() };
-            // Check received values to make sure the authority for them is the same as the sourceNodeId
-            //  (OR that WE are the authority). If not... it means we received values after changing the watcher,
-            //      which requires us to ignore the other values
-            // (Clients get most of their values from the same server, so they shouldn't require this check)
-            const self = this;
+            // NOTE: We need to ignore values if they're not who we're watching. That way, if we change the watcher, it's smooth and we don't partial data that might clobber with the data from the authority we are really using.
             if (!isClient()) {
                 measureBlock(function ignoreUnrequestedValues() {
                     for (let batch of batched) {
+                        function isWrongAuthority(path: string) {
+                            let watchingAuthorityId = remoteWatcher.getExistingWatchRemoteNodeId(path);
+                            // If we AREN'T watching it... it's actually fine, we can receive any values.
+                            //  When we start watching, those values will get clobbered.
+                            if (watchingAuthorityId === undefined) return false;
+                            return !areNodeIdsEqual(watchingAuthorityId, batch.sourceNodeId);
+                        }
                         batch.pathValues = batch.pathValues.filter(value => {
+                            // Authorities watch other authorities using path watches, And so one path can come from many authorities, so we just have to accept it no matter what, if it's from that type of source. 
+                            if (batch.authoritySyncPaths?.has(value.path)) return true;
+                            // NOTE: See the definition for lock count for why this check isn't checking all the possible cases. Essentially, locks is often empty, and that's intentional. However, the reverse should never be true, locks should never have values when lockCount is 0. 
                             if (value.lockCount === 0 && value.locks.length > 0) {
                                 console.error(red(`Ignoring value with invalid lockCount. Was ${value.lockCount}, but we have ${value.locks.length} locks. locks are optional, but lockCount isn't. We should never have locks without having lockCount set. ${debugPathValuePath(value)}`));
                                 return false;
                             }
 
-                            // If we are the authority, accept it (we are implicitly watching all of our own paths)
-                            if (nodePathAuthority.isSelfAuthority(value.path)) return true;
-                            let watchingAuthorityId = self.getExistingWatchRemoteNodeId(value.path);
-                            // If we AREN'T watching it... it's actually fine, we can receive any values.
-                            //  When we start watching, those values will get clobbered.
-                            if (watchingAuthorityId === undefined) return true;
-                            if (watchingAuthorityId === batch.sourceNodeId) return true;
+                            if (PathRouter.isSelfAuthority(value.path)) return true;
+                            if (isWrongAuthority(value.path)) return false;
                             // epochTimes are just indicators that the value has no value, and so are safe to sync
                             //  from any source. They won't cause future conflicts, because any other value overrides them.
                             //  - This might not be required?
@@ -264,402 +326,74 @@ class PathValueCommitter {
 
                             // Also warn, because... if we get a lot of these, there might be a bug.
                             //  A few when we change watches is possible, but it should be rare.
+                            let watchingAuthorityId = remoteWatcher.getExistingWatchRemoteNodeId(value.path);
                             auditLog("IGNORING VALUE FROM DIFFERENT AUTHORITY", { path: value.path, watchingAuthorityId, receivedFromAuthority: batch.sourceNodeId });
-                            console.warn(`Ignoring a value that was received from a different authority than the one we are watching. Path: ${value.path}, Watching Authority (should be source): ${watchingAuthorityId}, Actual Source Authority: ${batch.sourceNodeId}`);
                             return false;
                         });
-                    }
-                });
-            }
-
-            let specialInitialParentCausedRejections: WriteState[] = [];
-
-            // The first time we receive value for a watch from a specific node, we need to reject future
-            //  missing values (unless we are the authority). Otherwise bad rejected values can stick around,
-            //  because they are newer, and our new source doesn't know we have them.
-            measureBlock(function resetOnInitialTrigger() {
-                let receivedValues = new Map<string, Set<string>>();
-                for (let batch of batched) {
-                    for (let pathValue of batch.pathValues) {
-                        let path = pathValue.path;
-                        let hash = timeHash(pathValue.time);
-                        let paths = receivedValues.get(path);
-                        if (!paths) {
-                            paths = new Set();
-                            receivedValues.set(path, paths);
-                        }
-                        paths.add(hash);
-                    }
-                }
 
-                // TODO: Ugh... rejections on initialSync might the wrong solution. I think we have to just remove the values,
-                //  even though this makes debugging harder, in order to prevent the rejections from sticking around.
-
-                for (let batch of batched) {
-                    if (!batch.initialTrigger) continue;
-                    for (let pathValue of batch.pathValues) {
-                        initialTriggers.values.add(pathValue.path);
-                        auditLog("INITIAL SYNC", { path: pathValue.path, time: pathValue.time.time, sourceNodeId: batch.sourceNodeId });
-                        // We shouldn't receive the initialTrigger flag if we are the authority anyways
-                        if (nodePathAuthority.isSelfAuthority(pathValue.path)) continue;
-                        let history = authorityStorage.getValuePlusHistory(pathValue.path);
-                        for (let value of history) {
-                            // Definitely don't reset our predictions. We will reset those ourselves
-                            if (isOurPrediction(value)) continue;
-                            // Only reset values after us, otherwise we might clear valid history that
-                            //  we JUST received.
-                            if (compareTime(value.time, pathValue.time) <= 0) continue;
-
-                            // If we just received it, accept it.
-                            //  - Otherwise we might reject a valid older value, and then when the newer value is rejected
-                            //      the server won't tell us about the older value again, and so we will think no values
-                            //      are accepted, when the older value IS accepted.
-                            // BUG: In theory couldn't we receive an outdated value, then reconnect, then receive the correct
-                            //  value, and have them all batched together? In which case, we shouldn't accept the value.
-                            //  I'm not really sure how to fix this though. We kind of need to just handle adding the values
-                            //  and the rejections in order, instead of batching them, but that is much slower, and the race
-                            //  condition will likely be extremely rare (and only happen on disconnections anyways).
-                            if (receivedValues.get(pathValue.path)?.has(timeHash(pathValue.time))) continue;
-
-                            specialInitialParentCausedRejections.push({
-                                path: value.path,
-                                time: value.time,
-                                isValid: false,
-                                isTransparent: !!value.isTransparent,
-                                reason: "future missing on resync (initial trigger)",
-                            });
+                        for (let value of batch.initialTriggers.values) {
+                            if (isWrongAuthority(value)) {
+                                batch.initialTriggers.values.delete(value);
+                            }
                         }
-                    }
-                    // IF the parent is synced, but we don't receive a path value, then invalidate the
-                    //  old PathValue (as this means it is invalid, but the remote doesn't know to
-                    //  tell us it is invalid, as we only watched the parent, not the specific path).
-                    //  - If the values become unrejected in the future, we will receive them again, which
-                    //      will cause their valid state to be changed to accepted.
-                    //  - Also, if another batch sets the value, it will override our valid state write,
-                    //      as we are writing this in a special array will is always handled first.
-                    if (batch.parentsSynced) {
-                        for (let parentPath of batch.parentsSynced) {
-                            auditLog("INITIAL PARENT SYNC", { path: parentPath, sourceNodeId: batch.sourceNodeId });
-                            let allChildPaths = authorityStorage.getPathsFromParent(parentPath) || [];
-                            let childChecker = remoteWatcher.getChildPatchWatchChecker({
-                                parentPath,
-                                nodeId: batch.sourceNodeId
-                            });
-                            for (let childPath of allChildPaths) {
-                                if (!childChecker.isWatchedByNodeId(childPath)) continue;
-                                // We shouldn't receive the initialTrigger flag if we are the authority anyways.
-                                if (nodePathAuthority.isSelfAuthority(childPath)) continue;
-                                initialTriggers.parentPaths.add(childPath);
-
-                                // NOTE: On initial trigger, we send all child paths, so they should be
-                                //  in the values list, if they exist. AND if we receive them after,
-                                //  they will clobber this rejection
-                                //  - See PathWatcher.watchPath, where when !noInitialTrigger (aka, initialTrigger),
-                                //      we get all paths via authorityStorage.getPathsFromParent, and send their values.
-                                if (initialTriggers.values.has(childPath)) continue;
-
-                                let values = authorityStorage.getValuePlusHistory(childPath);
-                                if (values.length === 0) continue;
-
-                                values = values.filter(value => {
-                                    // DO NOT reject epoch values. For missing values, we still need a signal, and the parent
-                                    //  sync won't send this, so we need to NOT reject them.
-                                    if (compareTime(value.time, epochTime) === 0) {
-                                        return false;
-                                    }
-                                    return true;
-                                });
-                                if (values.length === 0) continue;
-
-                                console.log(self.getExistingWatchRemoteNodeId(childPath));
-                                for (let value of values) {
-                                    specialInitialParentCausedRejections.push({
-                                        path: value.path,
-                                        time: value.time,
-                                        isValid: false,
-                                        isTransparent: !!value.isTransparent,
-                                        reason: "child missing (initial trigger)",
-                                    });
-                                }
+                        for (let parentPath of batch.initialTriggers.parentPaths) {
+                            if (isWrongAuthority(parentPath)) {
+                                batch.initialTriggers.parentPaths.delete(parentPath);
                             }
                         }
                     }
-                }
-            });
-
-            let pathValues = batched.map(x => x.pathValues).flat();
-            let parentsSynced = batched.map(x => x.parentsSynced ?? []).flat();
-            let parentSyncedSources = new Map<string, string[]>();
-            for (let batch of batched) {
-                if (!batch.parentsSynced) continue;
-                for (let parentSynced of batch.parentsSynced) {
-                    let sources = parentSyncedSources.get(parentSynced);
-                    if (!sources) {
-                        sources = [];
-                        parentSyncedSources.set(parentSynced, sources);
-                    }
-                    sources.push(batch.sourceNodeId);
-                }
-            }
-
-
-            // Ignore outdated PathValues if we are the authority.
-            //  - This CAN leave our servers in a bad state. However...
-            //      1) We will resync with the disk eventually (in a few hours), fixing the state
-            //          (IF some servers accepted the values, and other ignored them).
-            //      1.1) We should also audit between PathValueServers, which should fix the state
-            //          MUCH faster.
-            //      2) If we DON'T ignore these values, we might end up rejecting an archived value,
-            //          which breaks a lot of values, and which cannot be automatically recovered from.
-            measureBlock(function discardInvalidGenesisValues() {
-                let maxAge = Date.now() - MAX_ACCEPTED_CHANGE_AGE;
-                // NOTE: We could add less to future age here, but it's fine. This is the time
-                //  a services (not a client, as services validate clients) can write in the future,
-                //  which will prevent writes on that path until after that time has passed.
-                let maxFutureAge = Date.now() + MAX_ACCEPTED_CHANGE_AGE;
-                pathValues = pathValues.filter(value => {
-                    if (!nodePathAuthority.isSelfAuthority(value.path)) return true;
-                    if (value.time.time < maxAge) {
-                        auditLog("DISCARD INVALID GENESIS VALUE", { path: value.path, time: value.time.time });
-                        console.error(red(`Ignoring a value that is ${formatTime(Date.now() - value.time.time)} pass change time, which would break assumptions made by archives / locks about change stability. ${debugPathValuePath(value)}`));
-                        return false;
-                    }
-                    if (value.time.time > maxFutureAge) {
-                        auditLog("DISCARD INVALID FUTURE VALUE", { path: value.path, time: value.time.time });
-                        console.error(red(`Ignoring a value that is ${formatTime(value.time.time - Date.now())} in the future, which would break writes to that path until that time passes. ${debugPathValuePath(value)}`));
-                        return false;
-                    }
-                    return true;
                 });
-            });
-
-            // NOTE: Values MUST be synchronously ingested, otherwise our "initialTrigger" resetting will break things.
-            this.ingestValues(pathValues, parentsSynced, parentSyncedSources, initialTriggers, specialInitialParentCausedRejections);
-            // Wait a microtick to allow triggered watchers to finish.
-            await Promise.resolve();
-            if (!isNode()) {
-                // Wait a frame, to allow painting to happen. (requestAnimationFrame waits until before paint,
-                //  so we have to wait twice).
-                await new Promise(resolve => requestAnimationFrame(resolve));
-                await new Promise(resolve => requestAnimationFrame(resolve));
             }
-        }
-    );
-
-    @measureFnc
-    public ingestValues(
-        pathValues: PathValue[],
-        parentsSynced: string[] | undefined,
-        parentSyncedSources: Map<string, string[]> | undefined,
-        initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
-        specialInitialParentCausedRejections?: WriteState[],
-    ): void {
-        if (pathValues.length === 0 && !parentsSynced?.length) return;
-
-        let now = Date.now();
 
-        // NOTE: specialInitialParentCausedRejections is a bit dangerous, because it can reject golden values.
-        //  This means we have to ingest it immediately, otherwise authorityStorage might gc path values
-        //  (that are before a golden value), when rejections might remove that golden value.
-        if (specialInitialParentCausedRejections?.length) {
-            this.ingestValidStates(specialInitialParentCausedRejections);
-        }
-
-        authorityStorage.ingestValues(pathValues, parentsSynced, parentSyncedSources);
-
-        // NOTE: This doesn't seem to be where we lag, so we aren't tracking it anymore.
-        for (let value of pathValues) {
-            let lag = now - value.time.time;
-            trackLag(now, lag);
-        }
-
-        // NOTE: Also triggers rejections of watched paths
-        this.ingestValidStates([], parentsSynced, undefined, pathValues, initialTriggers);
-    }
-
-    // NOTE: Technically, because we trigger all remotes any time a value changes, even
-    //  if it was changed by a remote, this could result in excessive notifications. However,
-    //  the only watchers are ourself, and remotes that consider us an authority.
-    //  AND, if we are an authority, we aren't asking any remotes for valid states. So...
-    //  it works out, with authorities receiving few/none external triggerWatchersForValidStateChange calls,
-    //  and non-authorities having few/no remote watchers.
-    @measureFnc
-    public ingestValidStates(
-        remoteLocksChanged: WriteState[],
-        parentsSynced?: string[],
-        recomputeValidState?: "recomputeValidState",
-        alsoPathValues?: PathValue[],
-        initialTriggers?: { values: Set<string>; parentPaths: Set<string> }
-    ) {
-        if (remoteLocksChanged.length === 0 && !parentsSynced?.length && !alsoPathValues?.length) return;
-
-        let now = Date.now();
-
-        let valuesChanged = new Set<PathValue>();
-
-        if (alsoPathValues) {
-            let complexNewValues: PathValue[] = [];
-            // TODO: We can avoid values being updated in the updateValidStatesFromCache call,
-            //  and then also being used in setWriteValidState. This might save some time.
-            //  It is rare though, as valid states are mostly only for function predictions,
-            //  and usually arrive after the ValuePaths? Maybe...
-            writeValidStorage.updateValidStatesFromCache(alsoPathValues);
-            for (let pathValue of alsoPathValues) {
-                // All the values are assumed to have changed
-                valuesChanged.add(pathValue);
-                if (pathValue.locks.length > 0) {
-                    complexNewValues.push(pathValue);
-                    continue;
-                }
-                let testValidState: WriteState = {
-                    path: pathValue.path,
-                    isValid: !!pathValue.valid,
-                    time: pathValue.time,
-                    isTransparent: pathValue.isTransparent || pathValue.canGCValue || false,
-                };
-                let callbacks = lockToCallback.getValidStateChangedTriggers(testValidState);
-                if (callbacks.some(x => typeof x !== "string")) {
-                    complexNewValues.push(pathValue);
-                    continue;
-                }
-                // If we get here, it is a simple value, with nothing watching it, and no possibility of
-                //  being rejected, so just mark it as valid.
-                writeValidStorage.setWriteValidState(testValidState);
-            }
-            if (complexNewValues.length > 0) {
-                let newValidStates = writeValidStorage.computeValidStates(complexNewValues, now);
-                for (let change of newValidStates) {
-                    remoteLocksChanged.push(change);
-                }
-            }
-        }
 
-        {
-            let allWrites = new Set<WriteState>();
-            let pendingWriteChanges = new Set<WriteState>(remoteLocksChanged);
-            while (pendingWriteChanges.size > 0) {
-                let curWriteChanges = pendingWriteChanges;
-                pendingWriteChanges = new Set();
-
-                let triggeredSelfChanges = new Set<PathValue[]>();
-
-                for (let changedWrite of curWriteChanges) {
-                    // NOTE: Predicted values are ingested, causing them to have a valid state. SO,
-                    //  if they are invalidated, a change will occur here (via the invalid value being forwarded,
-                    //  which is convert to triggering a valid state).
-                    let changed = true;
-
-                    // If we are recomputing it, don't use the changedWrite value, and assume it has changed,
-                    //  recomputing it always.
-                    if (!recomputeValidState) {
-                        changed = writeValidStorage.setWriteValidState(changedWrite);
+            // path => sourceNodeId
+            let parentSyncs = new Map<string, string>();
+
+            // We need to do a bit of work to properly clear path values that are from old initial triggers. As if we receive two initial triggers, they need to clobber each other. And if we collapse it, we lose that information. So we have to do that here. 
+            let finalResults = new Map<string, {
+                path: string;
+                pathValues: PathValue[];
+                initialTrigger: boolean;
+                batchIndex: number;
+            }>();
+
+            for (let batchIndex = 0; batchIndex < batched.length; batchIndex++) {
+                let batch = batched[batchIndex];
+                for (let pathValue of batch.pathValues) {
+                    let initialTrigger = batch.initialTriggers.values.has(pathValue.path);
+                    let results = finalResults.get(pathValue.path);
+                    if (!results) {
+                        results = {
+                            path: pathValue.path,
+                            pathValues: [],
+                            initialTrigger: false,
+                            batchIndex: -1,
+                        };
+                        finalResults.set(pathValue.path, results);
                     }
-                    if (changed) {
-                        // NOTE: If the value is invalid, the writer will get both the invalid value, AND a valid state change.
-                        //  This is unintentional, due to the fact that most watchers aren't watching for valid state changes,
-                        //  BUT, the writer might not be watching the path! But, if the writer is watching the path...
-                        //  we send both values, because at this don't know if it is the writer not not.
-                        //  - We can tell via looking to see if it is a valid state watcher, but... that happens in a different
-                        //      function, and... optimizing the reject path shouldn't matter, as rejections should rarely
-                        //      happen anyways...
-                        let valueChanged = authorityStorage.getValueAtTime(changedWrite.path, changedWrite.time);
-                        if (valueChanged) {
-                            valuesChanged.add(valueChanged);
-                        }
-                        // Have to send the previous value, as the caller might not have it (as we might start
-                        //  rejecting everything, result in the latest valid state being WAY in the past, to the point
-                        //  where it is impossible for any caller to have the value synced!
-                        if (!changedWrite.isValid) {
-                            let prevValue = authorityStorage.getValueBeforeTime(changedWrite.path, changedWrite.time);
-                            if (prevValue) {
-                                valuesChanged.add(prevValue);
-                            }
-                        }
-                    }
-
-                    allWrites.add(changedWrite);
-
-                    let callbacks = lockToCallback.getValidStateChangedTriggers(changedWrite);
-                    for (let valuesWatching of callbacks) {
-                        if (typeof valuesWatching !== "string") {
-                            triggeredSelfChanges.add(valuesWatching);
-                        }
-                    }
-                }
-
-                // TODO: I'm not sure if we need to sort these before updating them?
-                //  If we experience rejections and then undoing of rejections... we might need to...
-                let flatTriggeredChanges = Array.from(new Set(Array.from(triggeredSelfChanges).flat()));
-                let changes = writeValidStorage.computeValidStates(flatTriggeredChanges, now, "alreadyWatching");
-                for (let change of changes) {
-                    pendingWriteChanges.add(change);
-                }
-                // Anything triggered needs to be recomputed
-                recomputeValidState = "recomputeValidState";
-            }
-
-            // Handle valid state watches (mostly other authorities)
-            let validChangesPerNode = new Map<string, WriteState[]>();
-            for (let write of allWrites) {
-                let callbacks = lockToCallback.getValidStateChangedTriggers(write);
-                for (let valuesWatching of callbacks) {
-                    if (typeof valuesWatching !== "string") continue;
-                    let nodeId = valuesWatching;
-                    let changes = validChangesPerNode.get(nodeId);
-                    if (!changes) {
-                        changes = [];
-                        validChangesPerNode.set(nodeId, changes);
+                    let isContinuedBatch = results.batchIndex === batchIndex;
+                    results.batchIndex = batchIndex;
+                    if (initialTrigger && !isContinuedBatch) {
+                        results.pathValues = [];
+                        results.initialTrigger = true;
                     }
-                    changes.push(write);
+                    results.pathValues.push(pathValue);
                 }
-            }
-            // NOTE: Valid watchers are for lock watching of other authorities. The others nodes don't
-            //  want our values, and cannot properly ingest them (as they wouldn't be able to recursively resolve
-            //  them, they would be too large, etc, etc).
-            for (let [nodeId, validStates] of validChangesPerNode) {
-                if (validStates.length > 0) {
-                    logErrors(PathValueController.nodes[nodeId].onValidChange(validStates));
+                for (let parentPath of batch.initialTriggers.parentPaths) {
+                    parentSyncs.set(parentPath, batch.sourceNodeId);
                 }
             }
-        }
 
-        // Path watchers are for clients (ex FunctionRunner, browser, server scripts, etc),
-        //  which might be on other machines.
-        pathWatcher.triggerValuesChanged(valuesChanged, parentsSynced, initialTriggers);
-    }
-}
+            let parentPaths = new Set(parentSyncs.keys());
+            let initialValues = new Set(Array.from(finalResults.values()).filter(x => x.initialTrigger).map(x => x.path));
 
-let lagWindowSize = 2000;
-let lastLagInfo = {
-    value: createStatsValue(),
-    startTime: Date.now(),
-};
-let currentLagInfo = {
-    value: createStatsValue(),
-    startTime: Date.now(),
-};
-export function getReceiveLag() {
-    let stats = lastLagInfo.value;
-    if (stats.sum === 0) return undefined;
-    let top = getStatsTop(stats);
-    if (!top.topHeavy) {
-        return `${formatTime(stats.sum / stats.count)} TCP LAG`;
-    }
-    let bottomValue = stats.sum - top.value;
-    let bottomCount = stats.count - top.count;
-    return `${formatTime(top.value / top.count)} * ${formatNumber(top.count)} + ${formatTime(bottomValue / bottomCount)} * ${formatNumber(bottomCount)} LAG`;
-}
-registerMeasureInfo(getReceiveLag);
-function trackLag(now: number, lag: number) {
-    if (now - currentLagInfo.startTime > lagWindowSize) {
-        lastLagInfo = currentLagInfo;
-        currentLagInfo = {
-            value: createStatsValue(),
-            startTime: now,
-        };
-    }
-    addToStatsValue(currentLagInfo.value, lag);
+            validStateComputer.ingestValuesAndValidStates({
+                pathValues: Array.from(finalResults.values()).map(x => x.pathValues).flat(),
+                parentSyncs: Array.from(parentSyncs.entries()).map(([parentPath, sourceNodeId]) => ({ parentPath, sourceNodeId })),
+                initialTriggers: { values: initialValues, parentPaths: parentPaths },
+            });
+        },
+    );
 }
 
-
 export const pathValueCommitter = new PathValueCommitter();