querysub 0.2.0

package/src/0-path-value-core/PathController.ts

@@ -0,0 +1 @@
+export { LOCAL_DOMAIN } from "./NodePathAuthorities";

package/src/0-path-value-core/PathValueCommitter.ts

@@ -0,0 +1,565 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { delay, batchFunction } from "socket-function/src/batching";
+import { markArrayAsSplitable } from "socket-function/src/fixLargeNetworkCalls";
+import { isNode, timeInSecond } from "socket-function/src/misc";
+import { measureBlock, measureFnc, registerMeasureInfo } from "socket-function/src/profiling/measure";
+import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
+import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
+import { getPathFromStr } from "../path";
+import { nodePathAuthority, pathValueAuthority2 } from "./NodePathAuthorities";
+import { PathValueController } from "./PathValueController";
+import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath } from "./pathValueCore";
+import debugbreak from "debugbreak";
+import { red } from "socket-function/src/formatting/logColors";
+import { registerDynamicResource } from "../diagnostics/trackResources";
+import { StatsValue, addToStats, addToStatsValue, createStatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import { formatStats } from "socket-function/src/profiling/statsFormat";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { isClient } from "../config2";
+import { remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+
+/*
+- commitWrites <= creating writes
+    - ingestValues <= forwardWrites
+        - ingestValidStates <= forwarded valid states
+            = loop
+                - setValidState
+                - gather PathValue changes
+                - getValidStateChangedTriggers
+                - calculateInitialValidState
+            - REMOTE.ingestValidStates => forwarded valid states
+            - pathWatcher.triggerValuesChanged => forwardWrites + local clients
+    - broadcastValues
+*/
+
+class PathValueCommitter {
+    private pendingCommits = new Set<Promise<unknown>>();
+    private addCommitPromise(promise: Promise<unknown>) {
+        logErrors(promise);
+        this.pendingCommits.add(promise);
+        void promise.finally(() => this.pendingCommits.delete(promise));
+    }
+    public async waitForValuesToCommit() {
+        await Promise.all(this.pendingCommits);
+        // HACK: Wait a bit more, for the websocket to send the values (and for some batching, etc, etc)
+        //await new Promise(resolve => setTimeout(resolve, 100));
+        await delay("afterio");
+    }
+
+    @measureFnc
+    // Returns true if all the writes are accepted by some node (this should almost always be the case)
+    public commitValues(values: PathValue[], predictWrites: "predictWrites" | undefined): void {
+        if (values.length === 0) return;
+        ActionsHistory.OnWrite(values);
+
+        let now = Date.now();
+
+        // Validate the writes are in order (and not too old), just in case some delay in our code
+        //  causes them to be too old before we even get to send them! (Or if addWrites is called
+        //  from an endpoint?)
+        {
+            let maxAge = now - MAX_ACCEPTED_CHANGE_AGE;
+            let prevTime = 0;
+            for (let pathValue of values) {
+                if (pathValue.time.time < maxAge) {
+                    // NOTE: LIKELY caused by synchronizing taking too long, which is likely due to the PathValueServer massively lagging
+                    //  - Check the caller proxyWatcher startTime vs lastSyncTime, to see how long it took from start to sync
+                    //      (assuming the caller is a commitFunction, and not just a general watcher. If it is a general
+                    //      watcher it might have taken a really long time to run, in which case check the watchFunction).
+                    let message = `MAX_CHANGE_AGE_EXCEEDED! Cannot commit write, that is before the max age ${pathValue.time.time} < ${maxAge}. Acceping this write would result in changes in the past that wouldn't propagate correctly`;
+                    console.error(red(message));
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(message);
+                }
+                if (pathValue.time.time < prevTime) {
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(`PathValues must be in increasing order, but they were not.`);
+                }
+                if (pathValue.time.version % 1 !== 0) {
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(`PathValues must have an integer version, but they did not. Path: ${pathValue.path}, Time: ${pathValue.time.time}, Version: ${pathValue.time.version}`);
+                }
+                prevTime = pathValue.time.time;
+            }
+        }
+
+        let pathValuesToIngest: PathValue[] = [];
+        if (predictWrites) {
+            pathValuesToIngest = values;
+        } else {
+            pathValuesToIngest = values.filter(pathValue => pathValueAuthority2.isSelfAuthority(pathValue.path));
+        }
+
+        // NOTE: Error out early on the client, so we get better errors AND so we can avoid even ingesting the values
+        // TODO: Move this error to PathValueProxyWatcher, so it can be thrown on the exact line that does the write.
+        //  - This can be down by caching path (probably just the domain), and telling PathValueProxyWatcher
+        //      to throw on any accesses to it.
+        //      - PathValueProxyWatcher could even populate this cache itself on reads, so it will probably always
+        //          thrown on the first write attempt.
+        //          (It could even populate on the first domain, so it would always populate on the first write
+        //          on the current domain).
+        if (!isNode()) {
+            let remoteWrites = values.filter(pathValue => !pathValueAuthority2.isSelfAuthority(pathValue.path));
+            if (remoteWrites.length > 0) {
+                throw new Error(`Cannot commit writes to paths that are not local to this node. Paths: ${remoteWrites.map(x => x.path).join(", ")}`);
+            }
+        }
+
+        // NOTE: This function will just ignore writes we aren't an authority on, so we can just give it everything.
+        this.ingestValues(pathValuesToIngest, undefined, undefined);
+
+        // If all values are local, then there is nothing to broadcast
+        if (values.some(x => !pathValueAuthority2.isLocalPath(x.path))) {
+            this.addCommitPromise(this.broadcastValues(new Set(values)));
+        }
+    }
+
+    private broadcastValues = batchFunction(
+        { delay: 10, throttleWindow: 500 },
+        async function internal_forwardWrites(valuesBatched: Set<PathValue>[]) {
+            let values = new Set(valuesBatched.flatMap(x => Array.from(x)));
+
+            let valuesPerOtherAuthority = new Map<string, PathValue[]>();
+            for (let pathValue of values) {
+                if (isDebugLogEnabled()) {
+                    debugLog("CREATE VALUE", { path: pathValue.path, time: pathValue.time.time });
+                }
+
+                let otherAuthorities = await pathValueAuthority2.getWriteNodes(pathValue.path);
+                otherAuthorities = otherAuthorities.filter(x => !isOwnNodeId(x));
+                if (otherAuthorities.length === 0 && !pathValueAuthority2.isSelfAuthority(pathValue.path)) {
+                    throw new Error(`There are no authorities for path ${pathValue.path}. The write will be lost.`);
+                }
+                for (let otherAuthority of otherAuthorities) {
+                    let values = valuesPerOtherAuthority.get(otherAuthority);
+                    if (!values) {
+                        values = [];
+                        valuesPerOtherAuthority.set(otherAuthority, values);
+                    }
+                    values.push(pathValue);
+                }
+            }
+
+            // for (let [authorityId, values] of valuesPerOtherAuthority) {
+            //     console.log(`Sending ${values.length} values to ${authorityId} (${pathValueAuthority2.getAuthorityPaths(authorityId)?.map(x => pathValueAuthority2.getArchiveDirectory(x)).join(" | ")})`);
+            //     // for (let value of values) {
+            //     //     console.log("    " + value.path);
+            //     // }
+            // }
+
+            // Don't send to bad nodes for 60 seconds
+            const nodeIgnoreTime = Date.now() - 1000 * 60;
+            let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
+                let disconnected = SocketFunction.getLastDisconnectTime(otherAuthority);
+                if (disconnected && disconnected > nodeIgnoreTime) {
+                    // If it disconnected recently... don't send to it for a little bit, so we don't spend
+                    //  all of our time spamming disconnected nodes
+                    return;
+                }
+
+                let isTrusted = await isTrustedByNode(otherAuthority);
+                if (!isTrusted) {
+                    throw new Error(`Tried to write to paths on authorities not trusted by us. You probably need to call a function instead of directly writing to the server schema. Authority: ${otherAuthority}, Paths: ${values.map(x => getPathFromStr(x.path).join(".")).join(", ")}`);
+                }
+
+                // NOTE: We don't retry on failure, because... we broadcasted it anyways, AND this is supposed
+                //  to be on a trusted node (which should always be a server), so... either our network went
+                //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
+                //  writes are going to too old and therefore rejected by the time a server is up anyways.
+                markArrayAsSplitable(values);
+                const { Querysub } = await import("../4-querysub/Querysub");
+                let serializedValues = await pathValueSerializer.serialize(values, { compress: Querysub.COMPRESS_NETWORK });
+                diskLog(`Send PathValues to server`, { valueCount: values.length, targetId: otherAuthority, });
+                let forwardPromise = PathValueController.nodes[otherAuthority].forwardWrites(
+                    serializedValues,
+                    undefined,
+                    "watchLocks"
+                );
+                pathValueCommitter.addCommitPromise(forwardPromise);
+                logErrors(forwardPromise);
+                // NOTE: Usually we don't need watch the locks for our values because upon
+                //  forwarding them the server will automatically watch them. However...
+                //  if the call errors out, we can't know if we are watching them, so
+                //  we have to explicitly find a server to tell us if our writes go through
+                //  (we can't even just drop our writes, as we don't differentiate predicted
+                //  and real writes, so we can't tell if it got through and was sent back,
+                //  or if it was never committed!)
+                forwardPromise.catch(() => {
+
+                    // TODO: For the first few seconds, look for new writers and retry. However,
+                    //      after the write is too old, just log the error to the user, as
+                    //      at that point even if a writer appears the write will be too stale.
+
+                    // Watch the readLocks, so we know if the values are valid or not (this
+                    //  code has retry logic).
+                    //  NOTE: Technically if this talks to another authority, and then the original authority
+                    //      comes back, we might be syncing valid states from multiple authorities, and as
+                    //      they aren't timestamped, we could receive an older value later. Which should
+                    //      correct itself, but... we would also be subscribed to multiple authorities,
+                    //      for the same path, which can result in thrashing. But... this probably won't happen...
+                    lockWatcher.watchValueLocks(values);
+                });
+            });
+            // await, so "waitForValuesToCommit" works correctly
+            await Promise.all(promises.map(x => timeoutToUndefined(timeInSecond * 30, errorToUndefined(x))));
+        }
+    );
+
+
+    private getExistingWatchRemoteNodeId = (path: string): string | undefined => undefined;
+    public setGetRemoteWatchNodeId(fnc: (path: string) => string | undefined) {
+        this.getExistingWatchRemoteNodeId = fnc;
+    }
+
+    /** When ingesting remote values we batch a bit. Otherwise a stream of values will often result in WAY too many
+     *      clientside evaluations. The network will already add a lot of delay, so this shouldn't be noticeable.
+     *      - AND, with the throttleWindow, we generally won't even be waiting anyways.
+     *      - The delay is 16ms, as we won't render to the screen faster than that anyways, so handling updates
+     *          at a rate faster than that will just pointlessly add lag.
+     */
+    public ingestRemoteValues = batchFunction(
+        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValues" },
+        async (batched: {
+            pathValues: PathValue[],
+            parentsSynced?: string[];
+            sourceNodeId: string;
+            initialTrigger?: "initialTrigger";
+        }[]) => {
+            let initialTriggers = { values: new Set<PathValue>(), parentPaths: new Set<string>() };
+            // Check received values to make sure the authority for them is the same as the sourceNodeId
+            //  (OR that WE are the authority). If not... it means we received values after changing the watcher,
+            //      which requires us to ignore the other values
+            // (Clients get most of their values from the same server, so they shouldn't require this check)
+            if (!isClient()) {
+                const self = this;
+                measureBlock(function ignoreUnrequestedValues() {
+                    for (let batch of batched) {
+                        batch.pathValues = batch.pathValues.filter(value => {
+                            if (value.lockCount === 0 && value.locks.length > 0) {
+                                console.error(red(`Ignoring value with invalid lockCount. Was ${value.lockCount}, but we have ${value.locks.length} locks. locks are optional, but lockCount isn't. We should never have locks without having lockCount set. ${debugPathValuePath(value)}`));
+                                return false;
+                            }
+
+                            // If we are the authority, accept it (we are implicitly watching all of our own paths)
+                            if (nodePathAuthority.isSelfAuthority(value.path)) return true;
+                            let watchingAuthorityId = self.getExistingWatchRemoteNodeId(value.path);
+                            // If we AREN'T watching it... it's actually fine, we can receive any values.
+                            //  When we start watching, those values will get clobbered.
+                            if (watchingAuthorityId === undefined) return true;
+                            if (watchingAuthorityId === batch.sourceNodeId) return true;
+                            // Also warn, because... if we get a lot of these, there might be a bug.
+                            //  A few when we change watches is possible, but it should be rare.
+                            console.warn(`Ignoring a value that was received from a different authority than the one we are watching. Path: ${value.path}, Watching Authority (should be source): ${watchingAuthorityId}, Actual Source Authority: ${batch.sourceNodeId}`);
+                            return false;
+                        });
+                    }
+                });
+            }
+
+            // The first time we receive value for a watch from a specific node, we need to clobber our history
+            //  (unless we are the authority). Otherwise bad rejected values can stick around,
+            //  because they are newer, and our new source doesn't know we have them.
+            measureBlock(function resetOnInitialTrigger() {
+                for (let batch of batched) {
+                    if (!batch.initialTrigger) continue;
+                    for (let pathValue of batch.pathValues) {
+                        initialTriggers.values.add(pathValue);
+                        authorityStorage.resetForInitialTrigger(pathValue.path);
+                    }
+                    // Reset child paths as well, otherwise extra keys that have been deleted
+                    //  don't get properly removed.
+                    if (batch.parentsSynced) {
+                        for (let parentPath of batch.parentsSynced) {
+                            initialTriggers.parentPaths.add(parentPath);
+                            let allChildPaths = authorityStorage.getPathsFromParent(parentPath) || [];
+                            let childChecker = remoteWatcher.getChildPatchWatchChecker({
+                                parentPath,
+                                nodeId: batch.sourceNodeId
+                            });
+                            for (let childPath of allChildPaths) {
+                                if (childChecker.isWatchedByNodeId(childPath)) {
+                                    authorityStorage.resetForInitialTrigger(childPath);
+                                }
+                            }
+                        }
+                    }
+                }
+            });
+
+            let pathValues = batched.map(x => x.pathValues).flat();
+            let parentsSynced = batched.map(x => x.parentsSynced ?? []).flat();
+            let parentSyncedSources = new Map<string, string[]>();
+            for (let batch of batched) {
+                if (!batch.parentsSynced) continue;
+                for (let parentSynced of batch.parentsSynced) {
+                    let sources = parentSyncedSources.get(parentSynced);
+                    if (!sources) {
+                        sources = [];
+                        parentSyncedSources.set(parentSynced, sources);
+                    }
+                    sources.push(batch.sourceNodeId);
+                }
+            }
+
+            // Ignore outdated PathValues if we are the authority.
+            //  - This CAN leave our servers in a bad state. However...
+            //      1) We will resync with the disk eventually (in a few hours), fixing the state
+            //          (IF some servers accepted the values, and other ignored them).
+            //      1.1) We should also audit between PathValueServers, which should fix the state
+            //          MUCH faster.
+            //      2) If we DON'T ignore these values, we might end up rejecting an archived value,
+            //          which breaks a lot of values, and which cannot be automatically recovered from.
+            measureBlock(function discardInvalidGenesisValues() {
+                let maxAge = Date.now() - MAX_ACCEPTED_CHANGE_AGE;
+                // NOTE: We could add less to future age here, but it's fine. This is the time
+                //  a services (not a client, as services validate clients) can write in the future,
+                //  which will prevent writes on that path until after that time has passed.
+                let maxFutureAge = Date.now() + MAX_ACCEPTED_CHANGE_AGE;
+                pathValues = pathValues.filter(value => {
+                    if (!nodePathAuthority.isSelfAuthority(value.path)) return true;
+                    if (value.time.time < maxAge) {
+                        console.error(red(`Ignoring a value that is ${formatTime(Date.now() - value.time.time)} pass change time, which would break assumptions made by archives / locks about change stability. ${debugPathValuePath(value)}`));
+                        return false;
+                    }
+                    if (value.time.time > maxFutureAge) {
+                        console.error(red(`Ignoring a value that is ${formatTime(value.time.time - Date.now())} in the future, which would break writes to that path until that time passes. ${debugPathValuePath(value)}`));
+                        return false;
+                    }
+                    return true;
+                });
+            });
+
+            // NOTE: Values MUST be synchronously ingested, otherwise our "initialTrigger" resetting will break things.
+            this.ingestValues(pathValues, parentsSynced, parentSyncedSources, initialTriggers);
+            // Wait a microtick to allow triggered watchers to finish.
+            await Promise.resolve();
+            if (!isNode()) {
+                // Wait a frame, to allow painting to happen. (requestAnimationFrame waits until before paint,
+                //  so we have to wait twice).
+                await new Promise(resolve => requestAnimationFrame(resolve));
+                await new Promise(resolve => requestAnimationFrame(resolve));
+            }
+        }
+    );
+
+    @measureFnc
+    public ingestValues(
+        pathValues: PathValue[],
+        parentsSynced: string[] | undefined,
+        parentSyncedSources: Map<string, string[]> | undefined,
+        initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }
+    ): void {
+        if (pathValues.length === 0 && !parentsSynced?.length) return;
+
+        let now = Date.now();
+
+        authorityStorage.ingestValues(pathValues, parentsSynced, parentSyncedSources);
+
+        // NOTE: This doesn't seem to be where we lag, so we aren't tracking it anymore.
+        for (let value of pathValues) {
+            let lag = now - value.time.time;
+            trackLag(now, lag);
+        }
+
+        // NOTE: Also triggers rejections of watched paths
+        this.ingestValidStates([], parentsSynced, undefined, pathValues, initialTriggers);
+    }
+
+    // NOTE: Technically, because we trigger all remotes any time a value changes, even
+    //  if it was changed by a remote, this could result in excessive notifications. However,
+    //  the only watchers are ourself, and remotes that consider us an authority.
+    //  AND, if we are an authority, we aren't asking any remotes for valid states. So...
+    //  it works out, with authorities receiving few/none external triggerWatchersForValidStateChange calls,
+    //  and non-authorities having few/no remote watchers.
+    @measureFnc
+    public ingestValidStates(
+        remoteLocksChanged: WriteState[],
+        parentsSynced?: string[],
+        recomputeValidState?: "recomputeValidState",
+        alsoPathValues?: PathValue[],
+        initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }
+    ) {
+        if (remoteLocksChanged.length === 0 && !parentsSynced?.length && !alsoPathValues?.length) return;
+
+        if (alsoPathValues?.some(x => x.path.includes("#BREAK"))) {
+            debugger;
+        }
+
+        let now = Date.now();
+
+        let valuesChanged = new Set<PathValue>();
+
+        if (alsoPathValues) {
+            let complexNewValues: PathValue[] = [];
+            // TODO: We can avoid values being updated in the updateValidStatesFromCache call,
+            //  and then also being used in setWriteValidState. This might save some time.
+            //  It is rare though, as valid states are mostly only for function predictions,
+            //  and usually arrive after the ValuePaths? Maybe...
+            writeValidStorage.updateValidStatesFromCache(alsoPathValues);
+            for (let pathValue of alsoPathValues) {
+                // All the values are assumed to have changed
+                valuesChanged.add(pathValue);
+                if (pathValue.locks.length > 0) {
+                    complexNewValues.push(pathValue);
+                    continue;
+                }
+                let testValidState: WriteState = {
+                    path: pathValue.path,
+                    isValid: !!pathValue.valid,
+                    time: pathValue.time,
+                    isTransparent: pathValue.isTransparent || pathValue.canGCValue || false,
+                };
+                let callbacks = lockToCallback.getValidStateChangedTriggers(testValidState);
+                if (callbacks.some(x => typeof x !== "string")) {
+                    complexNewValues.push(pathValue);
+                    continue;
+                }
+                // If we get here, it is a simple value, with nothing watching it, and no possibility of
+                //  being rejected, so just mark it as valid.
+                writeValidStorage.setWriteValidState(testValidState);
+            }
+            if (complexNewValues.length > 0) {
+                let newValidStates = writeValidStorage.computeValidStates(complexNewValues, now);
+                for (let change of newValidStates) {
+                    remoteLocksChanged.push(change);
+                }
+            }
+        }
+
+        {
+            let allWrites = new Set<WriteState>();
+            let pendingWriteChanges = new Set<WriteState>(remoteLocksChanged);
+            while (pendingWriteChanges.size > 0) {
+                let curWriteChanges = pendingWriteChanges;
+                pendingWriteChanges = new Set();
+
+                let triggeredSelfChanges = new Set<PathValue[]>();
+
+                for (let changedWrite of curWriteChanges) {
+                    // NOTE: Predicted values are ingested, causing them to have a valid state. SO,
+                    //  if they are invalidated, a change will occur here (via the invalid value being forwarded,
+                    //  which is convert to triggering a valid state).
+                    let changed = true;
+
+                    // If we are recomputing it, don't use the changedWrite value, and assume it has changed,
+                    //  recomputing it always.
+                    if (!recomputeValidState) {
+                        changed = writeValidStorage.setWriteValidState(changedWrite);
+                    }
+                    if (changed) {
+                        // NOTE: If the value is invalid, the writer will get both the invalid value, AND a valid state change.
+                        //  This is unintentional, due to the fact that most watchers aren't watching for valid state changes,
+                        //  BUT, the writer might not be watching the path! But, if the writer is watching the path...
+                        //  we send both values, because at this don't know if it is the writer not not.
+                        //  - We can tell via looking to see if it is a valid state watcher, but... that happens in a different
+                        //      function, and... optimizing the reject path shouldn't matter, as rejections should rarely
+                        //      happen anyways...
+                        let valueChanged = authorityStorage.getValueAtTime(changedWrite.path, changedWrite.time);
+                        if (valueChanged) {
+                            valuesChanged.add(valueChanged);
+                        }
+                        // Have to send the previous value, as the caller might not have it (as we might start
+                        //  rejecting everything, result in the latest valid state being WAY in the past, to the point
+                        //  where it is impossible for any caller to have the value synced!
+                        if (!changedWrite.isValid) {
+                            let prevValue = authorityStorage.getValueBeforeTime(changedWrite.path, changedWrite.time);
+                            if (prevValue) {
+                                valuesChanged.add(prevValue);
+                            }
+                        }
+                    }
+
+                    allWrites.add(changedWrite);
+
+                    let callbacks = lockToCallback.getValidStateChangedTriggers(changedWrite);
+                    for (let valuesWatching of callbacks) {
+                        if (typeof valuesWatching !== "string") {
+                            triggeredSelfChanges.add(valuesWatching);
+                        }
+                    }
+                }
+
+                // TODO: I'm not sure if we need to sort these before updating them?
+                //  If we experience rejections and then undoing of rejections... we might need to...
+                let flatTriggeredChanges = Array.from(new Set(Array.from(triggeredSelfChanges).flat()));
+                let changes = writeValidStorage.computeValidStates(flatTriggeredChanges, now, "alreadyWatching");
+                for (let change of changes) {
+                    pendingWriteChanges.add(change);
+                }
+                // Anything triggered needs to be recomputed
+                recomputeValidState = "recomputeValidState";
+            }
+
+            // Handle valid state watches (mostly other authorities)
+            let validChangesPerNode = new Map<string, WriteState[]>();
+            for (let write of allWrites) {
+                let callbacks = lockToCallback.getValidStateChangedTriggers(write);
+                for (let valuesWatching of callbacks) {
+                    if (typeof valuesWatching !== "string") continue;
+                    let nodeId = valuesWatching;
+                    let changes = validChangesPerNode.get(nodeId);
+                    if (!changes) {
+                        changes = [];
+                        validChangesPerNode.set(nodeId, changes);
+                    }
+                    changes.push(write);
+                }
+            }
+            // NOTE: Valid watchers are for lock watching of other authorities. The others nodes don't
+            //  want our values, and cannot properly ingest them (as they wouldn't be able to recursively resolve
+            //  them, they would be too large, etc, etc).
+            for (let [nodeId, validStates] of validChangesPerNode) {
+                if (validStates.length > 0) {
+                    logErrors(PathValueController.nodes[nodeId].onValidChange(validStates));
+                }
+            }
+        }
+
+        // Path watchers are for clients (ex FunctionRunner, browser, server scripts, etc),
+        //  which might be on other machines.
+        pathWatcher.triggerValuesChanged(valuesChanged, parentsSynced, initialTriggers);
+    }
+}
+
+let lagWindowSize = 2000;
+let lastLagInfo = {
+    value: createStatsValue(),
+    startTime: Date.now(),
+};
+let currentLagInfo = {
+    value: createStatsValue(),
+    startTime: Date.now(),
+};
+export function getReceiveLag() {
+    let stats = lastLagInfo.value;
+    if (stats.sum === 0) return undefined;
+    let top = getStatsTop(stats);
+    if (!top.topHeavy) {
+        return `${formatTime(stats.sum / stats.count)} TCP LAG`;
+    }
+    let bottomValue = stats.sum - top.value;
+    let bottomCount = stats.count - top.count;
+    return `${formatTime(top.value / top.count)} * ${formatNumber(top.count)} + ${formatTime(bottomValue / bottomCount)} * ${formatNumber(bottomCount)} LAG`;
+}
+registerMeasureInfo(getReceiveLag);
+function trackLag(now: number, lag: number) {
+    if (now - currentLagInfo.startTime > lagWindowSize) {
+        lastLagInfo = currentLagInfo;
+        currentLagInfo = {
+            value: createStatsValue(),
+            startTime: now,
+        };
+    }
+    addToStatsValue(currentLagInfo.value, lag);
+}
+
+
+export const pathValueCommitter = new PathValueCommitter();