querysub 0.437.0 → 0.439.0

package/src/0-path-value-core/PathValueCommitter.ts

@@ -1,469 +1,469 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { delay, batchFunction } from "socket-function/src/batching";
-import { deepCloneJSON, isNode, timeInSecond } from "socket-function/src/misc";
-import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
-import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
-import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { ActionsHistory } from "../diagnostics/ActionsHistory";
-import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
-import { getPathFromStr, hack_stripPackedPath } from "../path";
-import { PathValueControllerBase } from "./PathValueController";
-import { PathRouter } from "./PathRouter";
-import { PathValue, MAX_ACCEPTED_CHANGE_AGE, WriteState, debugPathValuePath, compareTime, epochTime } from "./pathValueCore";
-import { validStateComputer } from "./ValidStateComputer";
-import debugbreak from "debugbreak";
-import { red } from "socket-function/src/formatting/logColors";
-import { isClient } from "../config2";
-import { auditLog, isDebugLogEnabled } from "./auditLogs";
-import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
-import { debugNodeId } from "../-c-identity/IdentityController";
-import { decodeNodeId } from "../-a-auth/certs";
-import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
-import { deepCloneCborx } from "../misc/cloneHelpers";
-import { removeRange } from "../rangeMath";
-import { registerShutdownHandler } from "../diagnostics/periodic";
-import { getCallFactory } from "socket-function/src/nodeCache";
-setImmediate(() => import("../1-path-client/RemoteWatcher"));
-setImmediate(() => import("../4-querysub/Querysub"));
-
-const MAX_SEND_TRY_COUNT = 3;
-
-// NOTE: This isn't very efficient, but it is safer. It's quadratic depending on the number of authorities for the server. Which shouldn't be so bad. I mean, why would we have four times redundancy? That's a lot. And even then, we're sending sixteen times the amount of traffic. That's fine, I guess. I think it's more reasonable for us to have two times redundancy, maybe even three times redundancy. With two times redundancy, it's only sending the value four times, which is reasonable. And the trade-off is there's basically no way that we can lose the data.
-//      - If we only send to one authority and then they rebroadcast it, it's possible that they die after receiving the value. If we send it to both value servers but they don't re-broadcast, it's possible that we die while broadcasting the values after having only sent a few. However, by sending it to both servers and having both servers send it to the other servers, it means that if we die while broadcasting it, it's fine. And if the server dies that we want to send it to, it's fine because we send it to both servers.
-const BROADCAST_TO_ALL_AUTHORITIES = true;
-
-export type BatchValues = {
-    pathValues: PathValue[],
-    parentsSynced?: string[];
-    sourceNodeId: string;
-    initialTrigger?: "initialTrigger";
-    authoritySync?: boolean;
-};
-
-export type RemoteValueAndValidState = {
-    sourceNodeId: string;
-    pathValues: PathValue[];
-    initialTriggers: { values: Set<string>; parentPaths: Set<string> };
-};
-
-class PathValueCommitter {
-    private pendingCommits = new Set<Promise<unknown>>();
-    private addCommitPromise(promise: Promise<unknown>) {
-        logErrors(promise);
-        this.pendingCommits.add(promise);
-        void promise.finally(() => this.pendingCommits.delete(promise));
-    }
-    public async waitForValuesToCommit() {
-        await this.broadcastValues({ values: new Set() });
-        await Promise.all(this.pendingCommits);
-        // HACK: Wait a bit more, for the websocket to send the values (and for some batching, etc, etc)
-        //await new Promise(resolve => setTimeout(resolve, 100));
-        await delay("afterio");
-    }
-
-    @measureFnc
-    // Returns true if all the writes are accepted by some node (this should almost always be the case)
-    public commitValues(values: PathValue[], predictWrites: "predictWrites" | undefined): void {
-        if (values.length === 0) return;
-        ActionsHistory.OnWrite(values);
-
-        let now = Date.now();
-
-        // Validate the writes are in order (and not too old), just in case some delay in our code
-        //  causes them to be too old before we even get to send them! (Or if addWrites is called
-        //  from an endpoint?)
-        {
-            let maxAge = now - MAX_ACCEPTED_CHANGE_AGE;
-            let prevTime = 0;
-            for (let pathValue of values) {
-                if (pathValue.time.time < maxAge) {
-                    // NOTE: LIKELY caused by synchronizing taking too long, which is likely due to the PathValueServer massively lagging
-                    //  - Check the caller proxyWatcher startTime vs lastSyncTime, to see how long it took from start to sync
-                    //      (assuming the caller is a commitFunction, and not just a general watcher. If it is a general
-                    //      watcher it might have taken a really long time to run, in which case check the watchFunction).
-                    let message = `MAX_CHANGE_AGE EXCEEDED! Cannot commit write, that is before the max age ${pathValue.time.time} < ${maxAge}. Acceping this write would result in changes in the past that wouldn't propagate correctly`;
-                    console.error(red(message));
-                    debugbreak(2);
-                    debugger;
-                    throw new Error(message);
-                }
-                if (pathValue.time.time < prevTime) {
-                    debugbreak(2);
-                    debugger;
-                    throw new Error(`PathValues must be in increasing order, but they were not.`);
-                }
-                if (pathValue.time.version % 1 !== 0) {
-                    debugbreak(2);
-                    debugger;
-                    throw new Error(`PathValues must have an integer version, but they did not. Path: ${pathValue.path}, Time: ${pathValue.time.time}, Version: ${pathValue.time.version}`);
-                }
-                prevTime = pathValue.time.time;
-            }
-        }
-
-        let pathValuesToIngest: PathValue[] = [];
-        if (predictWrites) {
-            pathValuesToIngest = values;
-        } else {
-            pathValuesToIngest = values.filter(pathValue => PathRouter.isSelfAuthority(pathValue.path));
-        }
-
-        // NOTE: Error out early on the client, so we get better errors AND so we can avoid even ingesting the values
-        // TODO: Move this error to PathValueProxyWatcher, so it can be thrown on the exact line that does the write.
-        //  - This can be down by caching path (probably just the domain), and telling PathValueProxyWatcher
-        //      to throw on any accesses to it.
-        //      - PathValueProxyWatcher could even populate this cache itself on reads, so it will probably always
-        //          thrown on the first write attempt.
-        //          (It could even populate on the first domain, so it would always populate on the first write
-        //          on the current domain).
-        if (!isNode()) {
-            let remoteWrites = values.filter(pathValue => !PathRouter.isSelfAuthority(pathValue.path));
-            if (remoteWrites.length > 0) {
-                throw new Error(`Cannot commit writes to paths that are not local to this node. Paths: ${remoteWrites.map(x => x.path).join(", ")}`);
-            }
-        }
-
-        // NOTE: This function will just ignore writes we aren't an authority on, so we can just give it everything.
-        validStateComputer.ingestValuesAndValidStates({
-            pathValues: pathValuesToIngest,
-            parentSyncs: [],
-            initialTriggers: { values: new Set(), parentPaths: new Set() },
-        });
-
-        let remoteValues = values.filter(x => !PathRouter.isSelfAuthority(x.path));
-
-        if (remoteValues.length > 0) {
-            this.addCommitPromise(this.broadcastValues({ values: new Set(remoteValues) }));
-        }
-
-        // If we're the authority on it, we should still share it with the other authorities. 
-        let stillShareValues = values.filter(x => !PathRouter.isLocalPath(x.path) && PathRouter.isSelfAuthority(x.path));
-        if (stillShareValues.length > 0) {
-            this.addCommitPromise(PathValueControllerBase.authorityShareValues({ pathValues: stillShareValues }));
-        }
-    }
-
-    private broadcastValues = batchFunction(
-        { delay: 10, throttleWindow: 500, noMeasure: true },
-        async function internal_forwardWrites(valuesBatched: {
-            values: Set<PathValue>;
-            tryCount?: number;
-        }[]) {
-            let values = new Set(valuesBatched.flatMap(x => Array.from(x.values)));
-            if (values.size === 0) return;
-            let tryCountPerValue = new Map<PathValue, number>();
-            for (let list of valuesBatched) {
-                for (let value of list.values) {
-                    if (!list.tryCount) continue;
-                    tryCountPerValue.set(value, list.tryCount);
-                }
-            }
-            if (tryCountPerValue.size > 0) {
-                console.info(`Syncing all authorities to ensure we have the latest values, due to failed writes.`);
-                await authorityLookup.syncAllNow();
-            }
-
-
-            let valuesPerOtherAuthority = new Map<string, PathValue[]>();
-            for (let pathValue of values) {
-                if (isDebugLogEnabled()) {
-                    // let valueStr: string | undefined;
-                    // if (typeof pathValue.value === "boolean" || typeof pathValue.value === "number") {
-                    //     valueStr = pathValue.value.toString();
-                    // } else if (typeof pathValue.value === "string") {
-                    //     valueStr = pathValue.value.slice(0, 100);
-                    //     if (valueStr.length < pathValue.value.length) {
-                    //         valueStr += `[+${pathValue.value.length - valueStr.length}]`;
-                    //     }
-                    // } else if (Buffer.isBuffer(pathValue.value)) {
-                    //     valueStr = `Buffer(${pathValue.value.length})`;
-                    // } else if (pathValue.value === null) {
-                    //     valueStr = "{null}";
-                    // } else {
-                    //     valueStr = `{${typeof pathValue.value}}`;
-                    // }
-                    auditLog("CREATE VALUE", {
-                        path: pathValue.path,
-                        timeId: pathValue.time.time,
-                        timeIdFull: pathValue.time,
-                        source: pathValue.source,
-                        transparent: pathValue.isTransparent,
-                        // value: valueStr,
-                        event: pathValue.event,
-                        // NOTE: I think it'd be too expensive to log all the locks. There's one thing to log every time we read a value or create a value, but to log all of the dependencies for each write path would be O(locksPerWrite * 2 * writes), which is just too much, and could easily result in ten thousand logs per write. I think it will be fine because if there's a rejection, then that will tell us the time ID that the rejected value was reading, Which we can infer to know at least one of the locks, as in the most important lock that that right had.
-                    });
-                }
-
-                let otherAuthorities = PathRouter.getAllAuthorities(pathValue.path);
-                otherAuthorities = otherAuthorities.filter(x => !isOwnNodeId(x.nodeId));
-                if (otherAuthorities.length === 0) {
-                    validStateComputer.ingestValuesAndValidStates({
-                        pathValues: [{
-                            ...pathValue,
-                            valid: false,
-                        }],
-                        parentSyncs: [],
-                        initialTriggers: { values: new Set(), parentPaths: new Set() },
-                    });
-                    console.error(`There are no authorities for path ${pathValue.path}. The write will be lost.`, {
-                        path: pathValue.path,
-                        timeId: pathValue.time.time,
-                        source: pathValue.source,
-                        otherAuthorities,
-                    });
-                    continue;
-                }
-                function sendToAuthority(otherAuthority: AuthorityEntry) {
-                    let values = valuesPerOtherAuthority.get(otherAuthority.nodeId);
-                    if (!values) {
-                        values = [];
-                        valuesPerOtherAuthority.set(otherAuthority.nodeId, values);
-                    }
-                    values.push(pathValue);
-                }
-                if (BROADCAST_TO_ALL_AUTHORITIES) {
-                    for (let otherAuthority of otherAuthorities) {
-                        sendToAuthority(otherAuthority);
-                    }
-                } else {
-                    // NOTE: Path routing is pretty strict. Once we disconnect, it'll remove it from the list of authorities. And if we can't talk to it, the retry logic will call us again.
-                    let otherAuthority = otherAuthorities[~~(Math.random() * otherAuthorities.length)];
-                    sendToAuthority(otherAuthority);
-                }
-            }
-
-            // Don't send to bad nodes for 60 seconds
-            const nodeIgnoreTime = Date.now() - 1000 * 60;
-
-            let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
-
-                let disconnected = SocketFunction.getLastDisconnectTime(otherAuthority);
-                if (!SocketFunction.isNodeConnected(otherAuthority) && disconnected && disconnected > nodeIgnoreTime) {
-                    // If it disconnected recently... don't send to it for a little bit, so we don't spend
-                    //  all of our time spamming disconnected nodes
-                    console.log(`disconnected at ${disconnected} > ${nodeIgnoreTime}`);
-                    return;
-                }
-
-                let isTrusted = await isTrustedByNode(otherAuthority);
-                if (!isTrusted) {
-                    console.log(`not trusted`);
-                    throw new Error(`Tried to write to paths on authorities not trusted by us. You probably need to call a function instead of directly writing to the server schema. Authority: ${otherAuthority}, Paths: ${values.map(x => getPathFromStr(x.path).join(".")).join(", ")}`);
-                }
-
-
-                // NOTE: We don't retry on failure, because... we broadcasted it anyways, AND this is supposed
-                //  to be on a trusted node (which should always be a server), so... either our network went
-                //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
-                //  writes are going to too old and therefore rejected by the time a server is up anyways.
-
-                let forwardPromise = PathValueControllerBase.createValues({
-                    nodeId: otherAuthority,
-                    pathValues: values,
-                });
-                logErrors(forwardPromise);
-                void forwardPromise.then(x => {
-                    if (x === "refused") {
-                        values = values.map(value => {
-                            console.info(`Rejecting value that was refused: ${debugPathValuePath(value)}`, {
-                                path: value.path,
-                                timeId: value.time.time,
-                            });
-                            return {
-                                ...value,
-                                valid: false,
-                            };
-                        });
-
-                        validStateComputer.ingestValuesAndValidStates({
-                            pathValues: values,
-                            parentSyncs: [],
-                            initialTriggers: { values: new Set(), parentPaths: new Set() },
-                        });
-                    }
-                });
-                // If we broadcast to all authorities, we can't also retry, as this means if one authority is down, we end up infinitely looping. And it breaks the servers and the logs. 
-                if (!BROADCAST_TO_ALL_AUTHORITIES) {
-                    void forwardPromise.catch(async error => {
-                        let byTryCount = new Map<number, PathValue[]>();
-                        for (let value of values) {
-                            let tryCount = tryCountPerValue.get(value) ?? 0;
-                            let arr = byTryCount.get(tryCount) ?? [];
-                            arr.push(value);
-                            byTryCount.set(tryCount, arr);
-                        }
-                        for (let [tryCount, values] of byTryCount.entries()) {
-                            tryCount++;
-                            if (tryCount > MAX_SEND_TRY_COUNT) {
-                                console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
-                                    error: error.message,
-                                    otherAuthority,
-                                    count: values.length,
-                                });
-                                continue;
-                            }
-                            for (let value of values) {
-                                console.error(`Retrying to send values after ${tryCount} tries.`, {
-                                    error: error.message,
-                                    otherAuthority,
-                                    path: value.path,
-                                    timeId: value.time.time,
-                                });
-                            }
-                            void pathValueCommitter.broadcastValues({
-                                values: new Set(values),
-                                tryCount: tryCount,
-                            });
-                        }
-                    });
-                }
-
-                pathValueCommitter.addCommitPromise(forwardPromise);
-            });
-            // await, so "waitForValuesToCommit" works correctly
-            await Promise.all(promises.map(x => timeoutToUndefined(timeInSecond * 30, errorToUndefined(x))));
-        }
-    );
-
-
-    public ingestRemoteValuesAndValidStates = batchFunction(
-        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
-        async (batched: RemoteValueAndValidState[]) => {
-            const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
-
-            // NOTE: We need to ignore values if they're not who we're watching. That way, if we change the watcher, it's smooth and we don't partial data that might clobber with the data from the authority we are really using.
-            if (!isClient()) {
-                measureBlock(function ignoreUnrequestedValues() {
-                    for (let batch of batched) {
-                        function isWrongAuthority(path: string, value?: PathValue, type?: string) {
-                            let watchingAuthorityId = remoteWatcher.getExistingWatchRemoteNodeId(path);
-                            // If we AREN'T watching it... it's actually fine, we can receive any values.
-                            //  When we start watching, those values will get clobbered.
-                            if (watchingAuthorityId === undefined) return false;
-                            if (!areNodeIdsEqual(watchingAuthorityId, batch.sourceNodeId)) {
-                                let valueWatchNode = remoteWatcher.getValueWatchRemoteNodeId(path);
-                                if (!valueWatchNode || !areNodeIdsEqual(valueWatchNode, batch.sourceNodeId)) {
-                                    let candidates = PathRouter.getAllAuthorities(path);
-                                    require("debugbreak")(2);
-                                    debugger;
-                                    remoteWatcher.getExistingWatchRemoteNodeId(path);
-                                    console.warn(`Ignoring value from wrong authority. Should have been ${debugNodeId(watchingAuthorityId)}, but was received from ${debugNodeId(batch.sourceNodeId)}.`, {
-                                        path,
-                                        type,
-                                        timeId: value?.time.time,
-                                        source: value?.source,
-                                        sourceNodeId: debugNodeId(batch.sourceNodeId),
-                                        sourceNodeThreadId: decodeNodeId(batch.sourceNodeId)?.threadId,
-                                        watchingAuthorityId: debugNodeId(watchingAuthorityId),
-                                        watchingAuthorityNodeThreadId: decodeNodeId(watchingAuthorityId)?.threadId,
-                                        isTransparent: value?.isTransparent,
-                                    });
-                                }
-                                return true;
-                            }
-                            return false;
-                        }
-                        batch.pathValues = batch.pathValues.filter(value => {
-                            // NOTE: See the definition for lock count for why this check isn't checking all the possible cases. Essentially, locks is often empty, and that's intentional. However, the reverse should never be true, locks should never have values when lockCount is 0. 
-                            if (value.lockCount === 0 && value.locks.length > 0) {
-                                console.error(red(`Ignoring value with invalid lockCount. Was ${value.lockCount}, but we have ${value.locks.length} locks. locks are optional, but lockCount isn't. We should never have locks without having lockCount set. ${debugPathValuePath(value)}`));
-                                return false;
-                            }
-
-                            if (PathRouter.isSelfAuthority(value.path)) return true;
-                            if (isWrongAuthority(value.path, value, "value")) {
-                                return false;
-                            }
-                            return true;
-                        });
-
-                        // NOTE: If we are the authority on it, it means we're not going to have a remote watch node ID. And if we did, it would be ourselves. So this also implicitly filters out any initial triggers for values that we are the authority on. 
-                        for (let path of Array.from(batch.initialTriggers.values)) {
-                            if (isWrongAuthority(path, undefined, "initialTrigger")) {
-                                batch.initialTriggers.values.delete(path);
-                            }
-                        }
-                        for (let parentPath of Array.from(batch.initialTriggers.parentPaths)) {
-                            if (remoteWatcher.isFinalRemoteWatchPath({ parentPath, nodeId: batch.sourceNodeId })) {
-                                continue;
-                            }
-
-                            console.warn(`Ignoring parent path which we aren't watching. From ${debugNodeId(batch.sourceNodeId)}.`, {
-                                parentPath,
-                                sourceNodeId: debugNodeId(batch.sourceNodeId),
-                                sourceNodeThreadId: decodeNodeId(batch.sourceNodeId)?.threadId,
-                            });
-                            batch.initialTriggers.parentPaths.delete(parentPath);
-                        }
-                    }
-                });
-            }
-
-
-            // path => sourceNodeId
-            let parentSyncs = new Map<string, Set<string>>();
-
-            // We need to do a bit of work to properly clear path values that are from old initial triggers. As if we receive two initial triggers, they need to clobber each other. And if we collapse it, we lose that information. So we have to do that here. 
-            let finalResults = new Map<string, {
-                path: string;
-                pathValues: PathValue[];
-                initialTrigger: boolean;
-                batchIndex: number;
-            }>();
-
-            for (let batchIndex = 0; batchIndex < batched.length; batchIndex++) {
-                let batch = batched[batchIndex];
-                for (let pathValue of batch.pathValues) {
-                    let initialTrigger = batch.initialTriggers.values.has(pathValue.path);
-                    let results = finalResults.get(pathValue.path);
-                    if (!results) {
-                        results = {
-                            path: pathValue.path,
-                            pathValues: [],
-                            initialTrigger: false,
-                            batchIndex: -1,
-                        };
-                        finalResults.set(pathValue.path, results);
-                    }
-                    let isContinuedBatch = results.batchIndex === batchIndex;
-                    results.batchIndex = batchIndex;
-                    if (initialTrigger && !isContinuedBatch) {
-                        results.pathValues = [];
-                        results.initialTrigger = true;
-                    }
-                    results.pathValues.push(pathValue);
-                }
-                for (let parentPath of batch.initialTriggers.parentPaths) {
-                    let sourceNodeIds = parentSyncs.get(parentPath);
-                    if (!sourceNodeIds) {
-                        sourceNodeIds = new Set();
-                        parentSyncs.set(parentPath, sourceNodeIds);
-                    }
-                    sourceNodeIds.add(batch.sourceNodeId);
-                }
-            }
-
-            let parentPaths = new Set(parentSyncs.keys());
-            let initialValues = new Set(Array.from(finalResults.values()).filter(x => x.initialTrigger).map(x => x.path));
-
-            let parentSyncsList: { parentPath: string; sourceNodeId: string }[] = [];
-            for (let [parentPath, sourceNodeIds] of parentSyncs.entries()) {
-                for (let sourceNodeId of sourceNodeIds) {
-                    parentSyncsList.push({ parentPath, sourceNodeId });
-                }
-            }
-
-            validStateComputer.ingestValuesAndValidStates({
-                pathValues: Array.from(finalResults.values()).map(x => x.pathValues).flat(),
-                parentSyncs: parentSyncsList,
-                initialTriggers: { values: initialValues, parentPaths: parentPaths },
-            });
-        },
-    );
-}
-
-export const pathValueCommitter = new PathValueCommitter();
-
-registerShutdownHandler(async () => {
-    await pathValueCommitter.waitForValuesToCommit();
+import { SocketFunction } from "socket-function/SocketFunction";
+import { delay, batchFunction } from "socket-function/src/batching";
+import { deepCloneJSON, isNode, timeInSecond } from "socket-function/src/misc";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
+import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
+import { getPathFromStr, hack_stripPackedPath } from "../path";
+import { PathValueControllerBase } from "./PathValueController";
+import { PathRouter } from "./PathRouter";
+import { PathValue, MAX_ACCEPTED_CHANGE_AGE, WriteState, debugPathValuePath, compareTime, epochTime } from "./pathValueCore";
+import { validStateComputer } from "./ValidStateComputer";
+import debugbreak from "debugbreak";
+import { red } from "socket-function/src/formatting/logColors";
+import { isClient } from "../config2";
+import { auditLog, isDebugLogEnabled } from "./auditLogs";
+import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
+import { debugNodeId } from "../-c-identity/IdentityController";
+import { decodeNodeId } from "../-a-auth/certs";
+import { decodeParentFilter, encodeParentFilter } from "./hackedPackedPathParentFiltering";
+import { deepCloneCborx } from "../misc/cloneHelpers";
+import { removeRange } from "../rangeMath";
+import { registerShutdownHandler } from "../diagnostics/periodic";
+import { getCallFactory } from "socket-function/src/nodeCache";
+setImmediate(() => import("../1-path-client/RemoteWatcher"));
+setImmediate(() => import("../4-querysub/Querysub"));
+
+const MAX_SEND_TRY_COUNT = 3;
+
+// NOTE: This isn't very efficient, but it is safer. It's quadratic depending on the number of authorities for the server. Which shouldn't be so bad. I mean, why would we have four times redundancy? That's a lot. And even then, we're sending sixteen times the amount of traffic. That's fine, I guess. I think it's more reasonable for us to have two times redundancy, maybe even three times redundancy. With two times redundancy, it's only sending the value four times, which is reasonable. And the trade-off is there's basically no way that we can lose the data.
+//      - If we only send to one authority and then they rebroadcast it, it's possible that they die after receiving the value. If we send it to both value servers but they don't re-broadcast, it's possible that we die while broadcasting the values after having only sent a few. However, by sending it to both servers and having both servers send it to the other servers, it means that if we die while broadcasting it, it's fine. And if the server dies that we want to send it to, it's fine because we send it to both servers.
+const BROADCAST_TO_ALL_AUTHORITIES = true;
+
+export type BatchValues = {
+    pathValues: PathValue[],
+    parentsSynced?: string[];
+    sourceNodeId: string;
+    initialTrigger?: "initialTrigger";
+    authoritySync?: boolean;
+};
+
+export type RemoteValueAndValidState = {
+    sourceNodeId: string;
+    pathValues: PathValue[];
+    initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+};
+
+class PathValueCommitter {
+    private pendingCommits = new Set<Promise<unknown>>();
+    private addCommitPromise(promise: Promise<unknown>) {
+        logErrors(promise);
+        this.pendingCommits.add(promise);
+        void promise.finally(() => this.pendingCommits.delete(promise));
+    }
+    public async waitForValuesToCommit() {
+        await this.broadcastValues({ values: new Set() });
+        await Promise.all(this.pendingCommits);
+        // HACK: Wait a bit more, for the websocket to send the values (and for some batching, etc, etc)
+        //await new Promise(resolve => setTimeout(resolve, 100));
+        await delay("afterio");
+    }
+
+    @measureFnc
+    // Returns true if all the writes are accepted by some node (this should almost always be the case)
+    public commitValues(values: PathValue[], predictWrites: "predictWrites" | undefined): void {
+        if (values.length === 0) return;
+        ActionsHistory.OnWrite(values);
+
+        let now = Date.now();
+
+        // Validate the writes are in order (and not too old), just in case some delay in our code
+        //  causes them to be too old before we even get to send them! (Or if addWrites is called
+        //  from an endpoint?)
+        {
+            let maxAge = now - MAX_ACCEPTED_CHANGE_AGE;
+            let prevTime = 0;
+            for (let pathValue of values) {
+                if (pathValue.time.time < maxAge) {
+                    // NOTE: LIKELY caused by synchronizing taking too long, which is likely due to the PathValueServer massively lagging
+                    //  - Check the caller proxyWatcher startTime vs lastSyncTime, to see how long it took from start to sync
+                    //      (assuming the caller is a commitFunction, and not just a general watcher. If it is a general
+                    //      watcher it might have taken a really long time to run, in which case check the watchFunction).
+                    let message = `MAX_CHANGE_AGE EXCEEDED! Cannot commit write, that is before the max age ${pathValue.time.time} < ${maxAge}. Acceping this write would result in changes in the past that wouldn't propagate correctly`;
+                    console.error(red(message));
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(message);
+                }
+                if (pathValue.time.time < prevTime) {
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(`PathValues must be in increasing order, but they were not.`);
+                }
+                if (pathValue.time.version % 1 !== 0) {
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(`PathValues must have an integer version, but they did not. Path: ${pathValue.path}, Time: ${pathValue.time.time}, Version: ${pathValue.time.version}`);
+                }
+                prevTime = pathValue.time.time;
+            }
+        }
+
+        let pathValuesToIngest: PathValue[] = [];
+        if (predictWrites) {
+            pathValuesToIngest = values;
+        } else {
+            pathValuesToIngest = values.filter(pathValue => PathRouter.isSelfAuthority(pathValue.path));
+        }
+
+        // NOTE: Error out early on the client, so we get better errors AND so we can avoid even ingesting the values
+        // TODO: Move this error to PathValueProxyWatcher, so it can be thrown on the exact line that does the write.
+        //  - This can be down by caching path (probably just the domain), and telling PathValueProxyWatcher
+        //      to throw on any accesses to it.
+        //      - PathValueProxyWatcher could even populate this cache itself on reads, so it will probably always
+        //          thrown on the first write attempt.
+        //          (It could even populate on the first domain, so it would always populate on the first write
+        //          on the current domain).
+        if (!isNode()) {
+            let remoteWrites = values.filter(pathValue => !PathRouter.isSelfAuthority(pathValue.path));
+            if (remoteWrites.length > 0) {
+                throw new Error(`Cannot commit writes to paths that are not local to this node. Paths: ${remoteWrites.map(x => x.path).join(", ")}`);
+            }
+        }
+
+        // NOTE: This function will just ignore writes we aren't an authority on, so we can just give it everything.
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: pathValuesToIngest,
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+        });
+
+        let remoteValues = values.filter(x => !PathRouter.isSelfAuthority(x.path));
+
+        if (remoteValues.length > 0) {
+            this.addCommitPromise(this.broadcastValues({ values: new Set(remoteValues) }));
+        }
+
+        // If we're the authority on it, we should still share it with the other authorities. 
+        let stillShareValues = values.filter(x => !PathRouter.isLocalPath(x.path) && PathRouter.isSelfAuthority(x.path));
+        if (stillShareValues.length > 0) {
+            this.addCommitPromise(PathValueControllerBase.authorityShareValues({ pathValues: stillShareValues }));
+        }
+    }
+
+    private broadcastValues = batchFunction(
+        { delay: 10, throttleWindow: 500, noMeasure: true },
+        async function internal_forwardWrites(valuesBatched: {
+            values: Set<PathValue>;
+            tryCount?: number;
+        }[]) {
+            let values = new Set(valuesBatched.flatMap(x => Array.from(x.values)));
+            if (values.size === 0) return;
+            let tryCountPerValue = new Map<PathValue, number>();
+            for (let list of valuesBatched) {
+                for (let value of list.values) {
+                    if (!list.tryCount) continue;
+                    tryCountPerValue.set(value, list.tryCount);
+                }
+            }
+            if (tryCountPerValue.size > 0) {
+                console.info(`Syncing all authorities to ensure we have the latest values, due to failed writes.`);
+                await authorityLookup.syncAllNow();
+            }
+
+
+            let valuesPerOtherAuthority = new Map<string, PathValue[]>();
+            for (let pathValue of values) {
+                if (isDebugLogEnabled()) {
+                    // let valueStr: string | undefined;
+                    // if (typeof pathValue.value === "boolean" || typeof pathValue.value === "number") {
+                    //     valueStr = pathValue.value.toString();
+                    // } else if (typeof pathValue.value === "string") {
+                    //     valueStr = pathValue.value.slice(0, 100);
+                    //     if (valueStr.length < pathValue.value.length) {
+                    //         valueStr += `[+${pathValue.value.length - valueStr.length}]`;
+                    //     }
+                    // } else if (Buffer.isBuffer(pathValue.value)) {
+                    //     valueStr = `Buffer(${pathValue.value.length})`;
+                    // } else if (pathValue.value === null) {
+                    //     valueStr = "{null}";
+                    // } else {
+                    //     valueStr = `{${typeof pathValue.value}}`;
+                    // }
+                    auditLog("CREATE VALUE", {
+                        path: pathValue.path,
+                        timeId: pathValue.time.time,
+                        timeIdFull: pathValue.time,
+                        source: pathValue.source,
+                        transparent: pathValue.isTransparent,
+                        // value: valueStr,
+                        event: pathValue.event,
+                        // NOTE: I think it'd be too expensive to log all the locks. There's one thing to log every time we read a value or create a value, but to log all of the dependencies for each write path would be O(locksPerWrite * 2 * writes), which is just too much, and could easily result in ten thousand logs per write. I think it will be fine because if there's a rejection, then that will tell us the time ID that the rejected value was reading, Which we can infer to know at least one of the locks, as in the most important lock that that right had.
+                    });
+                }
+
+                let otherAuthorities = PathRouter.getAllAuthorities(pathValue.path);
+                otherAuthorities = otherAuthorities.filter(x => !isOwnNodeId(x.nodeId));
+                if (otherAuthorities.length === 0) {
+                    validStateComputer.ingestValuesAndValidStates({
+                        pathValues: [{
+                            ...pathValue,
+                            valid: false,
+                        }],
+                        parentSyncs: [],
+                        initialTriggers: { values: new Set(), parentPaths: new Set() },
+                    });
+                    console.error(`There are no authorities for path ${pathValue.path}. The write will be lost.`, {
+                        path: pathValue.path,
+                        timeId: pathValue.time.time,
+                        source: pathValue.source,
+                        otherAuthorities,
+                    });
+                    continue;
+                }
+                function sendToAuthority(otherAuthority: AuthorityEntry) {
+                    let values = valuesPerOtherAuthority.get(otherAuthority.nodeId);
+                    if (!values) {
+                        values = [];
+                        valuesPerOtherAuthority.set(otherAuthority.nodeId, values);
+                    }
+                    values.push(pathValue);
+                }
+                if (BROADCAST_TO_ALL_AUTHORITIES) {
+                    for (let otherAuthority of otherAuthorities) {
+                        sendToAuthority(otherAuthority);
+                    }
+                } else {
+                    // NOTE: Path routing is pretty strict. Once we disconnect, it'll remove it from the list of authorities. And if we can't talk to it, the retry logic will call us again.
+                    let otherAuthority = otherAuthorities[~~(Math.random() * otherAuthorities.length)];
+                    sendToAuthority(otherAuthority);
+                }
+            }
+
+            // Don't send to bad nodes for 60 seconds
+            const nodeIgnoreTime = Date.now() - 1000 * 60;
+
+            let promises = Array.from(valuesPerOtherAuthority.entries()).map(async ([otherAuthority, values]) => {
+
+                let disconnected = SocketFunction.getLastDisconnectTime(otherAuthority);
+                if (!SocketFunction.isNodeConnected(otherAuthority) && disconnected && disconnected > nodeIgnoreTime) {
+                    // If it disconnected recently... don't send to it for a little bit, so we don't spend
+                    //  all of our time spamming disconnected nodes
+                    console.log(`disconnected at ${disconnected} > ${nodeIgnoreTime}`);
+                    return;
+                }
+
+                let isTrusted = await isTrustedByNode(otherAuthority);
+                if (!isTrusted) {
+                    console.log(`not trusted`);
+                    throw new Error(`Tried to write to paths on authorities not trusted by us. You probably need to call a function instead of directly writing to the server schema. Authority: ${otherAuthority}, Paths: ${values.map(x => getPathFromStr(x.path).join(".")).join(", ")}`);
+                }
+
+
+                // NOTE: We don't retry on failure, because... we broadcasted it anyways, AND this is supposed
+                //  to be on a trusted node (which should always be a server), so... either our network went
+                //  down, or all nodes went down. Either way, it likely won't fix itself quickly, and so these
+                //  writes are going to too old and therefore rejected by the time a server is up anyways.
+
+                let forwardPromise = PathValueControllerBase.createValues({
+                    nodeId: otherAuthority,
+                    pathValues: values,
+                });
+                logErrors(forwardPromise);
+                void forwardPromise.then(x => {
+                    if (x === "refused") {
+                        values = values.map(value => {
+                            console.info(`Rejecting value that was refused: ${debugPathValuePath(value)}`, {
+                                path: value.path,
+                                timeId: value.time.time,
+                            });
+                            return {
+                                ...value,
+                                valid: false,
+                            };
+                        });
+
+                        validStateComputer.ingestValuesAndValidStates({
+                            pathValues: values,
+                            parentSyncs: [],
+                            initialTriggers: { values: new Set(), parentPaths: new Set() },
+                        });
+                    }
+                });
+                // If we broadcast to all authorities, we can't also retry, as this means if one authority is down, we end up infinitely looping. And it breaks the servers and the logs. 
+                if (!BROADCAST_TO_ALL_AUTHORITIES) {
+                    void forwardPromise.catch(async error => {
+                        let byTryCount = new Map<number, PathValue[]>();
+                        for (let value of values) {
+                            let tryCount = tryCountPerValue.get(value) ?? 0;
+                            let arr = byTryCount.get(tryCount) ?? [];
+                            arr.push(value);
+                            byTryCount.set(tryCount, arr);
+                        }
+                        for (let [tryCount, values] of byTryCount.entries()) {
+                            tryCount++;
+                            if (tryCount > MAX_SEND_TRY_COUNT) {
+                                console.error(`Failed to send values after ${MAX_SEND_TRY_COUNT} tries. Giving up.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    count: values.length,
+                                });
+                                continue;
+                            }
+                            for (let value of values) {
+                                console.error(`Retrying to send values after ${tryCount} tries.`, {
+                                    error: error.message,
+                                    otherAuthority,
+                                    path: value.path,
+                                    timeId: value.time.time,
+                                });
+                            }
+                            void pathValueCommitter.broadcastValues({
+                                values: new Set(values),
+                                tryCount: tryCount,
+                            });
+                        }
+                    });
+                }
+
+                pathValueCommitter.addCommitPromise(forwardPromise);
+            });
+            // await, so "waitForValuesToCommit" works correctly
+            await Promise.all(promises.map(x => timeoutToUndefined(timeInSecond * 30, errorToUndefined(x))));
+        }
+    );
+
+
+    public ingestRemoteValuesAndValidStates = batchFunction(
+        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
+        async (batched: RemoteValueAndValidState[]) => {
+            const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
+
+            // NOTE: We need to ignore values if they're not who we're watching. That way, if we change the watcher, it's smooth and we don't partial data that might clobber with the data from the authority we are really using.
+            if (!isClient()) {
+                measureBlock(function ignoreUnrequestedValues() {
+                    for (let batch of batched) {
+                        function isWrongAuthority(path: string, value?: PathValue, type?: string) {
+                            let watchingAuthorityId = remoteWatcher.getExistingWatchRemoteNodeId(path);
+                            // If we AREN'T watching it... it's actually fine, we can receive any values.
+                            //  When we start watching, those values will get clobbered.
+                            if (watchingAuthorityId === undefined) return false;
+                            if (!areNodeIdsEqual(watchingAuthorityId, batch.sourceNodeId)) {
+                                let valueWatchNode = remoteWatcher.getValueWatchRemoteNodeId(path);
+                                if (!valueWatchNode || !areNodeIdsEqual(valueWatchNode, batch.sourceNodeId)) {
+                                    let candidates = PathRouter.getAllAuthorities(path);
+                                    require("debugbreak")(2);
+                                    debugger;
+                                    remoteWatcher.getExistingWatchRemoteNodeId(path);
+                                    console.warn(`Ignoring value from wrong authority. Should have been ${debugNodeId(watchingAuthorityId)}, but was received from ${debugNodeId(batch.sourceNodeId)}.`, {
+                                        path,
+                                        type,
+                                        timeId: value?.time.time,
+                                        source: value?.source,
+                                        sourceNodeId: debugNodeId(batch.sourceNodeId),
+                                        sourceNodeThreadId: decodeNodeId(batch.sourceNodeId)?.threadId,
+                                        watchingAuthorityId: debugNodeId(watchingAuthorityId),
+                                        watchingAuthorityNodeThreadId: decodeNodeId(watchingAuthorityId)?.threadId,
+                                        isTransparent: value?.isTransparent,
+                                    });
+                                }
+                                return true;
+                            }
+                            return false;
+                        }
+                        batch.pathValues = batch.pathValues.filter(value => {
+                            // NOTE: See the definition for lock count for why this check isn't checking all the possible cases. Essentially, locks is often empty, and that's intentional. However, the reverse should never be true, locks should never have values when lockCount is 0. 
+                            if (value.lockCount === 0 && value.locks.length > 0) {
+                                console.error(red(`Ignoring value with invalid lockCount. Was ${value.lockCount}, but we have ${value.locks.length} locks. locks are optional, but lockCount isn't. We should never have locks without having lockCount set. ${debugPathValuePath(value)}`));
+                                return false;
+                            }
+
+                            if (PathRouter.isSelfAuthority(value.path)) return true;
+                            if (isWrongAuthority(value.path, value, "value")) {
+                                return false;
+                            }
+                            return true;
+                        });
+
+                        // NOTE: If we are the authority on it, it means we're not going to have a remote watch node ID. And if we did, it would be ourselves. So this also implicitly filters out any initial triggers for values that we are the authority on. 
+                        for (let path of Array.from(batch.initialTriggers.values)) {
+                            if (isWrongAuthority(path, undefined, "initialTrigger")) {
+                                batch.initialTriggers.values.delete(path);
+                            }
+                        }
+                        for (let parentPath of Array.from(batch.initialTriggers.parentPaths)) {
+                            if (remoteWatcher.isFinalRemoteWatchPath({ parentPath, nodeId: batch.sourceNodeId })) {
+                                continue;
+                            }
+
+                            console.warn(`Ignoring parent path which we aren't watching. From ${debugNodeId(batch.sourceNodeId)}.`, {
+                                parentPath,
+                                sourceNodeId: debugNodeId(batch.sourceNodeId),
+                                sourceNodeThreadId: decodeNodeId(batch.sourceNodeId)?.threadId,
+                            });
+                            batch.initialTriggers.parentPaths.delete(parentPath);
+                        }
+                    }
+                });
+            }
+
+
+            // path => sourceNodeId
+            let parentSyncs = new Map<string, Set<string>>();
+
+            // We need to do a bit of work to properly clear path values that are from old initial triggers. As if we receive two initial triggers, they need to clobber each other. And if we collapse it, we lose that information. So we have to do that here. 
+            let finalResults = new Map<string, {
+                path: string;
+                pathValues: PathValue[];
+                initialTrigger: boolean;
+                batchIndex: number;
+            }>();
+
+            for (let batchIndex = 0; batchIndex < batched.length; batchIndex++) {
+                let batch = batched[batchIndex];
+                for (let pathValue of batch.pathValues) {
+                    let initialTrigger = batch.initialTriggers.values.has(pathValue.path);
+                    let results = finalResults.get(pathValue.path);
+                    if (!results) {
+                        results = {
+                            path: pathValue.path,
+                            pathValues: [],
+                            initialTrigger: false,
+                            batchIndex: -1,
+                        };
+                        finalResults.set(pathValue.path, results);
+                    }
+                    let isContinuedBatch = results.batchIndex === batchIndex;
+                    results.batchIndex = batchIndex;
+                    if (initialTrigger && !isContinuedBatch) {
+                        results.pathValues = [];
+                        results.initialTrigger = true;
+                    }
+                    results.pathValues.push(pathValue);
+                }
+                for (let parentPath of batch.initialTriggers.parentPaths) {
+                    let sourceNodeIds = parentSyncs.get(parentPath);
+                    if (!sourceNodeIds) {
+                        sourceNodeIds = new Set();
+                        parentSyncs.set(parentPath, sourceNodeIds);
+                    }
+                    sourceNodeIds.add(batch.sourceNodeId);
+                }
+            }
+
+            let parentPaths = new Set(parentSyncs.keys());
+            let initialValues = new Set(Array.from(finalResults.values()).filter(x => x.initialTrigger).map(x => x.path));
+
+            let parentSyncsList: { parentPath: string; sourceNodeId: string }[] = [];
+            for (let [parentPath, sourceNodeIds] of parentSyncs.entries()) {
+                for (let sourceNodeId of sourceNodeIds) {
+                    parentSyncsList.push({ parentPath, sourceNodeId });
+                }
+            }
+
+            validStateComputer.ingestValuesAndValidStates({
+                pathValues: Array.from(finalResults.values()).map(x => x.pathValues).flat(),
+                parentSyncs: parentSyncsList,
+                initialTriggers: { values: initialValues, parentPaths: parentPaths },
+            });
+        },
+    );
+}
+
+export const pathValueCommitter = new PathValueCommitter();
+
+registerShutdownHandler(async () => {
+    await pathValueCommitter.waitForValuesToCommit();
 });