querysub 0.402.0 → 0.404.0

package/src/1-path-client/pathValueClientWatcher.ts

@@ -1,815 +1,808 @@
-/**
-    Does everything needed to synchronize values with PathValueController, including
-        storing the values. While this can result in redundant storage, it helps keep
-        our strictly synchronized values, with values we just get from another node.
-        - We also don't consider rejections at all, and just assume there is no
-            contention in this file.
-        - Keys based on baseTime, so our values can be correctly clobbered by the remote.
-   
-        setValues
-            - For writing values
-        getValue
-            - For getting values
-        setWatches
-            - For knowing when values change
- */
-
-import { SocketFunction } from "socket-function/SocketFunction";
-import { cache, lazy } from "socket-function/src/caching";
-import { binarySearchIndex, isNode, recursiveFreeze, sort } from "socket-function/src/misc";
-import { logErrors } from "../errors";
-import { getParentPathStr, getPathFromStr, hack_stripPackedPath } from "../path";
-import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
-import { pathValueCommitter, PathValueController } from "../0-path-value-core/PathValueController";
-import { PathValue, Value, getNextTime, Time, ReadLock, pathWatcher, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig, decodeParentFilter, matchesParentRangeFilter } from "../0-path-value-core/pathValueCore";
-import { blue, green, red } from "socket-function/src/formatting/logColors";
-import { MaybePromise } from "socket-function/src/types";
-import { batchFunction, batchFunctionNone, runInfinitePoll } from "socket-function/src/batching";
-import { registerResource } from "../diagnostics/trackResources";
-import debugbreak from "debugbreak";
-import { ActionsHistory } from "../diagnostics/ActionsHistory";
-import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
-import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { PromiseObj } from "../promise";
-import { getOwnMachineId } from "../-a-auth/certs";
-import { remoteWatcher } from "./RemoteWatcher";
-import { heapTagObj } from "../diagnostics/heapTag";
-import { Querysub } from "../4-querysub/QuerysubController";
-import { isDevDebugbreak } from "../config";
-import { auditLog } from "../0-path-value-core/auditLogs";
-
-const pathValueCtor = heapTagObj("PathValue");
-
-export interface WatchSpecData {
-    paths: Set<string>;
-    pathSources: Set<PathValue> | undefined;
-    newParentsSynced: Set<string>;
-    extraReasons?: string[];
-}
-
-export interface WatchSpec {
-    debugName?: string;
-
-    // Run order is determined by the lowest [orderGroup, order]. It is HIGHLY recommended
-    //      to use orderGroup, with low values (0 is the default, so use 1 to run later,
-    //      and -1 to run earlier).
-    //      - Using a large number of unique orderGroups (for example, a unique group per callback),
-    //          will slow down callbacks.
-    //      - "order" is defaulted to a good default (the construct order), and so setting it will often
-    //          result in callbacks being given too high of a priority.
-    order?: number;
-    orderGroup?: number;
-
-    // callback is the key used to update watches, if the same callback is used
-    //  it won't start a new watch, but will instead update the paths used.
-    callback: (changed: WatchSpecData) => void;
-    unwatchEventPaths?: (paths: Set<string>) => void;
-    paths: Set<string>;
-    parentPaths: Set<string>;
-    /** For PathValueProxyWatcher. BUT, not in the Querysub routing service, or most backend middle-man type services. */
-    noInitialTrigger?: boolean;
-
-    // IMPORTANT! Any variables that should be persisted, must be copied over in updateUnwatches.
-    //  Otherwise we don't keep them. We do this for GC reasons (I think?).
-    //  - We might want to try reusing the same object... it should greatly reduce our GC overhead...
-    breakOnNextTrigger?: boolean;
-    triggerCount?: number;
-}
-
-let nextWatchSpecOrder = 1;
-export class ClientWatcher {
-    public static DEBUG_READS = false;
-    public static DEBUG_WRITES = false;
-    public static DEBUG_READS_EXPANDED = false;
-    public static DEBUG_WRITES_EXPANDED = false;
-    public static DEBUG_TRIGGERS?: "light" | "heavy";
-    public static DEBUG_SOURCES = false;
-
-    /** How long we keep watching, despite a value no longer being needed. */
-    public static WATCH_STICK_TIME = MAX_CHANGE_AGE * 2;
-    public static MAX_TRIGGER_TIME_BROWSER = 1000 * 5;
-    public static MAX_TRIGGER_TIME_NODEJS = 1000 * 300;
-
-
-    private valueFunctionWatchers = registerResource("paths|valueFunctionWatchers", new Map<string, Map<WatchSpec["callback"], WatchSpec>>());
-    // hack_stripPackedPath(path) =>
-    private parentValueFunctionWatchers = registerResource("paths|parentValueFunctionWatchers", new Map<string,
-        // path =>
-        Map<string, {
-            start: number;
-            end: number;
-            lookup: Map<WatchSpec["callback"], WatchSpec>
-        }>
-    >());
-    private allWatchers = registerResource("paths|clientWatcher.allWatchers", new Map<WatchSpec["callback"], WatchSpec>());
-
-    // path => expiryTime
-    private pendingUnwatches = registerResource("paths|pendingUnwatches", new Map<string, number>());
-    private pendingParentUnwatches = registerResource("paths|pendingParentUnwatches", new Map<string, number>());
-
-
-    private inLoop = false;
-    private isLoopSynchronous = false;
-    private pendingTriggered: Map<WatchSpec, WatchSpecData> | undefined;
-
-    private preTrigger(spec: WatchSpec) {
-        spec.triggerCount = (spec.triggerCount ?? 0) + 1;
-        if (spec.breakOnNextTrigger) {
-            spec.breakOnNextTrigger = false;
-            debugger;
-        }
-    }
-
-    private triggerWatcher(spec: WatchSpec, config?: { synchronous?: boolean }) {
-        this.preTrigger(spec);
-        let watchedTriggers = this.pendingTriggered = this.pendingTriggered || (new Map() as never);
-        let trigger = watchedTriggers.get(spec);
-        if (!trigger) {
-            trigger = {
-                pathSources: ClientWatcher.DEBUG_SOURCES ? new Set() : undefined,
-                newParentsSynced: new Set(),
-                paths: new Set(),
-            };
-            watchedTriggers.set(spec, trigger);
-        }
-        this.triggerWatchLoop(config);
-        return trigger;
-    }
-    private beforeTriggerDone: Promise<void> | undefined;
-    private onTriggerDone: Promise<void> | undefined;
-    public activeWatchSpec: WatchSpec | undefined;
-    private ignoreWatch: ((spec: WatchSpec, source: PathValue | string) => boolean) | undefined;
-    // Needed for typesafecss. Same as onTriggerDone, but fires first.
-    public waitForBeforeTriggerFinished() { return this.beforeTriggerDone; }
-    public waitForTriggerFinished() { return this.onTriggerDone; }
-    public isCurrentLoopSynchronous() { return this.isLoopSynchronous; }
-
-    @measureFnc
-    public localOnValueCallback(values: PathValue[], parentsSynced: string[]) {
-        if (values.length === 0 && parentsSynced.length === 0) return;
-        const onTrigger = (spec: WatchSpec, path: string, source?: PathValue) => {
-            if (
-                ClientWatcher.DEBUG_TRIGGERS === "heavy"
-                && !this.pendingTriggered?.has(spec)
-            ) {
-                let sourceName = "";
-                if (this.activeWatchSpec) {
-                    sourceName = ` (inside ${this.activeWatchSpec.debugName})`;
-                } else if (source?.source) {
-                    sourceName = ` (remote ${source.source.split("|").at(-1)})`;
-                }
-                console.log(`${blue("QUEUEING TRIGGER")} ${spec.debugName} ${sourceName}`);
-                console.log(`    DUE TO WRITE ${getPathFromStr(path).join(".")}`);
-            }
-        };
-
-        const triggerWatcher = (spec: WatchSpec, source: PathValue) => {
-            // Ignore self writes, otherwise local writes as simple as `x++`, will trigger an infinite loop.
-            if (this.ignoreWatch?.(spec, source)) return;
-            let trigger = this.triggerWatcher(spec);
-            if (trigger.pathSources) {
-                trigger.pathSources.add(source);
-            }
-            trigger.paths.add(source.path);
-            onTrigger(spec, source.path, source);
-        };
-        const triggerWatcherParent = (spec: WatchSpec, parent: string) => {
-            if (this.ignoreWatch?.(spec, parent)) return;
-            let trigger = this.triggerWatcher(spec);
-            trigger.newParentsSynced.add(parent);
-            onTrigger(spec, parent);
-        };
-        for (let value of values) {
-            let watchers = this.valueFunctionWatchers.get(value.path);
-            if (watchers) {
-                for (let watcher of watchers.values()) {
-                    triggerWatcher(watcher, value);
-                }
-            }
-            let parentPath = getParentPathStr(value.path);
-            let parentWatchers = this.parentValueFunctionWatchers.get(parentPath);
-            if (parentWatchers) {
-                // NOTE: We don't filter by path shard here
-                for (let { start, end, lookup } of parentWatchers.values()) {
-                    if (!matchesParentRangeFilter({ parentPath, fullPath: value.path, start, end })) {
-                        continue;
-                    }
-
-                    for (let watcher of lookup.values()) {
-                        triggerWatcher(watcher, value);
-                        triggerWatcherParent(watcher, parentPath);
-                    }
-                }
-            }
-        }
-        for (let path of parentsSynced) {
-            let basePath = hack_stripPackedPath(path);
-            let watchers = this.parentValueFunctionWatchers.get(basePath);
-            if (watchers) {
-                let realObj = watchers.get(path);
-                if (realObj) {
-                    for (let watcher of realObj.lookup.values()) {
-                        triggerWatcherParent(watcher, path);
-                    }
-                }
-            }
-        }
-        // Trigger all children of parents synced
-        // NOTE: This is the counterpointer to RemoteWatcher's removal of extra requests when we are already syncing the parent path. If it's not going to request them, we're not going to get the paths, so we need to trigger based on parents.
-        //  - We ignore hack paths here (that path in range requests in parent paths), because RemoteWatcher also ignores them (so hacked parent paths won't stop value path syncs there), so they will be received normally.
-        // NOTE: This will probably be slow, but we shouldn't be receiving parent syncs very frequently. Or maybe we will, and maybe this will just be incredibly slow. 
-        if (parentsSynced.length > 0) {
-            measureBlock(() => {
-                for (let parentPath of parentsSynced) {
-                    for (let [path, lookup] of this.valueFunctionWatchers.entries()) {
-                        if (!path.startsWith(parentPath)) continue;
-                        if (getParentPathStr(path) !== parentPath) continue;
-                        for (let watch of lookup.values()) {
-                            let trigger = this.triggerWatcher(watch);
-                            trigger.paths.add(path);
-                            onTrigger(watch, path);
-                        }
-                    }
-                }
-            }, "ClientWatcher|triggerChildrenOfParentsSynced");
-        }
-    }
-
-    public debugBreakOnNextTrigger() {
-        if (!this.activeWatchSpec) throw new Error(`Break on next trigger must be called inside of a clientWatcher trigger`);
-        this.activeWatchSpec.breakOnNextTrigger = true;
-    }
-
-    public explicitlyTriggerWatcher(callback: WatchSpec["callback"], config?: { synchronous?: boolean }) {
-        let watcher = this.allWatchers.get(callback);
-        if (!watcher) throw new Error(`No watcher found for callback ${callback.toString()}`);
-        this.triggerWatcher(watcher, config);
-    }
-
-    private triggerWatchLoop(config?: { synchronous?: boolean }) {
-        // A fairly standard trigger loop. Batch synchronous triggers for efficiency's sake
-        if (this.inLoop) return;
-        this.inLoop = true;
-        let beforeTriggerDone = new PromiseObj();
-        let triggerPromiseObj = new PromiseObj();
-        this.beforeTriggerDone = beforeTriggerDone.promise;
-        this.onTriggerDone = triggerPromiseObj.promise;
-        const doLoop = () => {
-            try {
-                this.innerTriggerLoop();
-            } finally {
-                this.inLoop = false;
-                this.isLoopSynchronous = false;
-                beforeTriggerDone.resolve();
-                triggerPromiseObj.resolve();
-                this.beforeTriggerDone = undefined;
-                this.onTriggerDone = undefined;
-            }
-        };
-        if (config?.synchronous) {
-            this.isLoopSynchronous = true;
-            doLoop();
-        } else {
-            void Promise.resolve().finally(doLoop);
-        }
-    }
-    @measureFnc
-    private innerTriggerLoop() {
-        let loopCount = 0;
-        let triggerCount = 0;
-        let groupStart = (
-            ClientWatcher.DEBUG_TRIGGERS === "light" && console.groupCollapsed
-            || ClientWatcher.DEBUG_TRIGGERS === "heavy" && console.group
-            || undefined
-        );
-
-        for (let callback of this.loopStartCallbacks) {
-            try {
-                callback();
-            } catch (e) {
-                logErrors(e);
-            }
-        }
-        let MAX_TRIGGER_TIME = isNode() ? ClientWatcher.MAX_TRIGGER_TIME_NODEJS : ClientWatcher.MAX_TRIGGER_TIME_BROWSER;
-
-        let time = Date.now();
-        while (true) {
-            let currentTriggered = this.pendingTriggered;
-            this.pendingTriggered = undefined;
-            if (!currentTriggered?.size) break;
-
-            if (ClientWatcher.DEBUG_TRIGGERS) {
-                console.log(" ");
-            }
-            groupStart?.(`${blue("TRIGGERS LOOP ITERATION")} ${loopCount++}`);
-
-            let sorted = Array.from(currentTriggered);
-            sorted.sort((a, b) => {
-                let orderDiff = (a[0].orderGroup ?? 0) - (b[0].orderGroup ?? 0);
-                if (orderDiff !== 0) return orderDiff;
-                return (a[0].order ?? 0) - (b[0].order ?? 0);
-            });
-            if (ClientWatcher.DEBUG_TRIGGERS) {
-                for (let [trigger] of sorted) {
-                    console.log("    " + trigger.debugName + " " + (trigger.orderGroup ?? 0) + "|" + (trigger.order ?? 0));
-                }
-                console.groupEnd();
-            }
-            let curIndex = 0;
-            this.ignoreWatch = (spec, source) => {
-                this.preTrigger(spec);
-                // Ignore it if we are handling it (self triggers are almost always just due to `x++`, and are virtually
-                //  never intended to actually self trigger. And if they are, then they can just detach with
-                //  Promise.resolve.finally(...))
-                // ALSO ignore if we are going to handle it. There's no reason to queue another watch for something
-                //  we are about to handle.
-                for (let checkIndex = curIndex; checkIndex < sorted.length; checkIndex++) {
-                    if (sorted[checkIndex][0] === spec) {
-                        let watchSpecData = sorted[checkIndex][1];
-                        if (typeof source === "string") {
-                            watchSpecData.newParentsSynced.add(source);
-                        } else {
-                            if (watchSpecData.pathSources) {
-                                watchSpecData.pathSources.add(source);
-                            }
-                            watchSpecData.paths.add(source.path);
-                        }
-                        return true;
-                    }
-                }
-
-                // If they care about the order, and it is higher than the current order, then insert it into the current
-                if (spec.order !== undefined && (
-                    (spec.order ?? 0) > (sorted[curIndex][0].order ?? 0)
-                    && (spec.orderGroup ?? 0) === (sorted[curIndex][0].orderGroup ?? 0)
-                    || (spec.orderGroup ?? 0) > (sorted[curIndex][0].orderGroup ?? 0)
-                )) {
-                    if (ClientWatcher.DEBUG_TRIGGERS === "heavy") {
-                        console.log("   (insert new trigger)");
-                        console.log("    " + spec.debugName + " " + (spec.orderGroup ?? 0) + "|" + (spec.order ?? 0));
-                    }
-                    // Find the last insert >= the current order (check for equals so we can insert at the end
-                    //      duplicate order values).
-                    let insertIndex = sorted.findLastIndex(x =>
-                        (spec.order ?? 0) >= (x[0].order ?? 0)
-                        && (spec.orderGroup ?? 0) === (x[0].orderGroup ?? 0)
-                        || (spec.orderGroup ?? 0) > (x[0].orderGroup ?? 0)
-                    );
-                    if (insertIndex === -1) {
-                        insertIndex = sorted.length - 1;
-                    }
-                    insertIndex++;
-                    let watchSpecData: WatchSpecData = {
-                        paths: new Set(),
-                        pathSources: ClientWatcher.DEBUG_SOURCES ? new Set() : undefined,
-                        newParentsSynced: new Set(),
-                    };
-                    if (typeof source === "string") {
-                        watchSpecData.newParentsSynced.add(source);
-                    } else {
-                        if (watchSpecData.pathSources) {
-                            watchSpecData.pathSources.add(source);
-                        }
-                        watchSpecData.paths.add(source.path);
-                    }
-                    sorted.splice(insertIndex, 0, [spec, watchSpecData]);
-                    return true;
-                }
-                return false;
-            };
-
-            let stoppedEarly = false;
-            for (; curIndex < sorted.length; curIndex++) {
-                let [watchSpec, data] = sorted[curIndex];
-                triggerCount++;
-                if (ClientWatcher.DEBUG_TRIGGERS === "heavy") {
-                    console.log(`${green("RUNNING TRIGGER")} ${watchSpec.debugName}`);
-                }
-
-                try {
-                    this.activeWatchSpec = watchSpec;
-                    watchSpec.callback(data);
-                } catch (e) {
-                    void Promise.resolve().finally(() => { throw e; });
-                } finally {
-                    this.activeWatchSpec = undefined;
-                }
-                if (
-                    Date.now() - time > MAX_TRIGGER_TIME
-                    && !isDevDebugbreak()
-                ) {
-                    stoppedEarly = true;
-                    break;
-                }
-            }
-            this.ignoreWatch = undefined;
-
-            if (ClientWatcher.DEBUG_TRIGGERS) {
-                console.log(" ");
-            }
-
-            if (stoppedEarly) {
-                const history = 10;
-                let remaining = (
-                    sorted
-                        .slice(curIndex, curIndex + history)
-                        .map(x => `${x[0].debugName} (${Array.from(x[1].paths).map(x => getPathFromStr(x).join(".")).join(" | ")})`)
-                        .join(", ")
-                );
-                console.error(red(`Too much time spent in trigger loop, aborting loop. Triggered ${triggerCount} triggers. Remaining triggers: ${remaining}`), triggerCount);
-                break;
-            }
-        }
-
-        let finalStats = { loopCount, triggerCount };
-        for (let callback of this.loopEndCallbacks) {
-            try {
-                callback(finalStats);
-            } catch (e) {
-                logErrors(e);
-            }
-        }
-
-        time = Date.now() - time;
-        if (ClientWatcher.DEBUG_TRIGGERS) {
-            console.log(`${green("TRIGGERS LOOP FINISHED")} ${loopCount} loops, ${triggerCount} triggers, in ${time}ms`);
-            console.log(" ");
-            console.log(" ");
-        }
-    }
-
-    private loopStartCallbacks = new Set<() => void>();
-    private loopEndCallbacks = new Set<(info: { loopCount: number; triggerCount: number; }) => void>();
-    public onLoopStart(callback: () => void) {
-        this.loopStartCallbacks.add(callback);
-    }
-    public onLoopEnd(
-        callback: (info: {
-            loopCount: number;
-            triggerCount: number;
-        }) => void
-    ) {
-        this.loopEndCallbacks.add(callback);
-    }
-
-    public unwatch(callback: WatchSpec["callback"]) {
-        this.updateUnwatches({
-            callback,
-            paths: new Set(),
-            parentPaths: new Set(),
-        });
-        this.allWatchers.delete(callback);
-    }
-
-    @measureFnc
-    private updateUnwatches(watchSpec: WatchSpec) {
-        const { callback, paths, parentPaths } = watchSpec;
-        let prevSpec = this.allWatchers.get(callback);
-        if (!prevSpec) return;
-        if (prevSpec) {
-            watchSpec.order = prevSpec.order;
-            watchSpec.triggerCount = prevSpec.triggerCount;
-            watchSpec.breakOnNextTrigger = prevSpec.breakOnNextTrigger;
-        }
-
-        let fullyUnwatchedPaths = new Set<string>();
-        let fullyUnwatchedParents = new Set<string>();
-
-        // Remove removed paths
-        for (let path of prevSpec.paths) {
-            if (paths.has(path)) continue;
-            let watchers = this.valueFunctionWatchers.get(path);
-            if (!watchers) continue;
-            watchers.delete(callback);
-            if (watchers.size === 0) {
-                fullyUnwatchedPaths.add(path);
-                this.valueFunctionWatchers.delete(path);
-            }
-        }
-
-        // Remove removed parents
-        for (let path of prevSpec.parentPaths) {
-            if (parentPaths.has(path)) continue;
-            let basePath = hack_stripPackedPath(path);
-            let watchersBase = this.parentValueFunctionWatchers.get(basePath);
-            if (!watchersBase) continue;
-            let watchers = watchersBase.get(path);
-            if (!watchers) continue;
-            watchers.lookup.delete(callback);
-            if (watchers.lookup.size === 0) {
-                watchersBase.delete(path);
-                fullyUnwatchedParents.add(path);
-            }
-            if (watchersBase.size === 0) {
-                this.parentValueFunctionWatchers.delete(basePath);
-            }
-        }
-
-        prevSpec.paths = paths;
-        prevSpec.parentPaths = parentPaths;
-        watchSpec = prevSpec;
-
-        let expiryTime = Date.now() + ClientWatcher.WATCH_STICK_TIME;
-        // Commented out, as this is showing up as a bit slow in profiling
-        // if (isDevDebugbreak()) {
-        //     for (let path of fullyUnwatchedPaths) {
-        //         auditLog("clientWatcher UNWATCH", { path });
-        //     }
-        //     for (let path of fullyUnwatchedParents) {
-        //         auditLog("clientWatcher UNWATCH PARENT", { path });
-        //     }
-        // }
-        for (let path of fullyUnwatchedPaths) {
-            this.pendingUnwatches.set(path, expiryTime);
-        }
-        for (let path of fullyUnwatchedParents) {
-            this.pendingParentUnwatches.set(path, expiryTime);
-        }
-        this.ensureUnwatching();
-    }
-    private ensureUnwatching = lazy(() => {
-        let waitTime = Math.max(1000 * 5, ClientWatcher.WATCH_STICK_TIME - 1000);
-        const self = this;
-        runInfinitePoll(waitTime, async function ensureUnwatching() {
-            let now = Date.now();
-            let pathsToUnwatch = new Set<string>();
-            let parentPathsToUnwatch = new Set<string>();
-            for (let [path, expiryTime] of self.pendingUnwatches) {
-                if (expiryTime < now) {
-                    self.pendingUnwatches.delete(path);
-                    pathsToUnwatch.add(path);
-                }
-            }
-            for (let [path, expiryTime] of self.pendingParentUnwatches) {
-                if (expiryTime < now) {
-                    self.pendingParentUnwatches.delete(path);
-                    parentPathsToUnwatch.add(path);
-                }
-            }
-            if (pathsToUnwatch.size > 0 || parentPathsToUnwatch.size > 0) {
-                //console.log(red("Unwatching paths"), pathsToUnwatch, parentPathsToUnwatch, Date.now());
-                let unwatchConfig: WatchConfig = { paths: Array.from(pathsToUnwatch), parentPaths: Array.from(parentPathsToUnwatch), };
-                // pathWatcher unwatches remotely as well
-                pathWatcher.unwatchPath({ callback: getOwnNodeId(), ...unwatchConfig });
-            }
-        });
-    });
-
-
-    // TODO: EVENTUALLY! We should accept a delta on the watches, which should allow for more efficient updates.
-
-    // NOTE: The callback is only called when synced values change (so if they are already synced, and don't change, the callback won't be called).
-    // NOTE: Doesn't call the callback until all requested values have finished their initial sync, to prevent too many callback callbacks.
-    //  - You can just fragment your writes into different components to isolate any slow loading parts, so partial
-    //      loading of data really isn't needed.
-    // NOTE: Takes ownership of paths and parentPaths, so... don't mutate them after calling this!
-    @measureFnc
-    public setWatches(watchSpec: WatchSpec) {
-        // NOTE: Yes, setWatches shows up as being slow in the profiler. I added measureBlock to every single section, and none of them showed up as being slow.
-        //  - My best guess is that after we clobber values in various lookups, but BEFORE we return, the destructor for those old resources runs. So their destruct time gets put in our profiler?
-        //      - Or maybe hack_stripPackedPath, decodeParentFilter, or other functions with heavy temporary allocations.
-        //  - measureBlock also has overhead (which is why I commented them all out), but... It's not nearly enough to account for this.
-
-
-        //measureBlock(() => {
-        watchSpec.order = watchSpec.order ?? nextWatchSpecOrder++;
-        //}, "ClientWatcher()|watchSpec.order = ...");
-
-        //measureBlock(() => {
-        this.updateUnwatches(watchSpec);
-        //}, "ClientWatcher()|updateUnwatches wrapper?");
-
-        //measureBlock(() => {
-        this.allWatchers.set(watchSpec.callback, watchSpec);
-        //}, "ClientWatcher()|setWatches|this.allWatcher.set");
-        //measureBlock(() => {
-        for (let path of watchSpec.paths) {
-            let watchers = this.valueFunctionWatchers.get(path);
-            if (!watchers) {
-                watchers = new Map();
-                this.valueFunctionWatchers.set(path, watchers);
-            }
-            watchers.set(watchSpec.callback, watchSpec);
-            this.pendingUnwatches.delete(path);
-        }
-        //}, "ClientWatcher()|setWatches|paths");
-        //measureBlock(() => {
-        for (let path of watchSpec.parentPaths) {
-            let basePath = hack_stripPackedPath(path);
-            let watchersBase = this.parentValueFunctionWatchers.get(basePath);
-            if (!watchersBase) {
-                watchersBase = new Map();
-                this.parentValueFunctionWatchers.set(basePath, watchersBase);
-            }
-            let watchers = watchersBase.get(path);
-            if (!watchers) {
-                let range = decodeParentFilter(path) || { start: 0, end: 1 };
-                watchers = {
-                    start: range.start,
-                    end: range.end,
-                    lookup: new Map()
-                };
-                watchersBase.set(path, watchers);
-            }
-            watchers.lookup.set(watchSpec.callback, watchSpec);
-            this.pendingParentUnwatches.delete(path);
-        }
-        //}, "ClientWatcher()|setWatches|parentPaths");
-
-        if (watchSpec.paths.size === 0 && watchSpec.parentPaths.size === 0) {
-            return;
-        }
-
-        //measureBlock(() => {
-        let pathsArray = Array.from(watchSpec.paths);
-        let parentPathsArray = Array.from(watchSpec.parentPaths);
-        let debugName = watchSpec.callback.name;
-        // Watch it locally as well, so when we get the value we know to trigger the client callback
-        pathWatcher.watchPath({
-            callback: getOwnNodeId(),
-            paths: pathsArray,
-            parentPaths: parentPathsArray,
-            debugName,
-            noInitialTrigger: watchSpec.noInitialTrigger,
-        });
-        remoteWatcher.watchLatest({
-            paths: pathsArray,
-            parentPaths: parentPathsArray,
-            debugName,
-        });
-        //}, "ClientWatcher()|setWatches|watchCalls");
-    }
-
-    private lastVersions = registerResource("paths|lastVersion", new Map<number, number>());
-    private getFreeVersionBase(time: number): number {
-        this.ensureCleaningUpVersions();
-        let lastVersion = this.lastVersions.get(time);
-        let nextVersion = lastVersion === undefined ? 0 : lastVersion + 1;
-        this.lastVersions.set(time, nextVersion);
-        return nextVersion;
-    }
-    private ensureCleaningUpVersions = lazy(() => {
-        const self = this;
-        const THRESHOLD = MAX_CHANGE_AGE * 2;
-        runInfinitePoll(THRESHOLD, function cleanUpVersions() {
-            let curThreshold = Date.now() - THRESHOLD;
-            for (let time of self.lastVersions.keys()) {
-                if (time < curThreshold) {
-                    self.lastVersions.delete(time);
-                }
-            }
-        });
-    });
-
-    /** Gets a free write time by incrementing the version, and then ensuring that version isn't reused for the given time. */
-    public getFreeWriteTime(time: Time | undefined): Time {
-        if (!time?.time) return getNextTime();
-        let version = this.getFreeVersionBase(time.time);
-        return { time: time.time, version, creatorId: getCreatorId() };
-    }
-
-    // IMPORTANT! The entire path from setValues => localOnValueCallback MUST be synchronous. If it is asynchronous we CANNOT
-    //      correctly prevent watchers from triggering themselves, which can result in either 2X the number of needed evaluations,
-    //      OR infinite evaluations.
-    // NOTE: This should be called RIGHT after you read the values, otherwise the lock that is created
-    //  may not depend on what was actually read.
-    // NOTE: If a server goes down, these writes may fail to commit (they will depend on each other, so
-    //  it will be either all or nothing).
-    // NOTE: This synchronously predicts the values, so you don't need to wait. When the promise resolves
-    //  the values will be committed and they won't belost (although it is still possible for them to become
-    //  rejected).
-    @measureFnc
-    public setValues(
-        config: {
-            values: Map<string, Value>;
-            forcedUndefinedWrites?: Set<string>;
-            eventPaths?: Set<string>;
-            locks: ReadLock[];
-            // NOTE: Use clientWatcher.getFreeWriteTime() to ensure this writeTime is valid
-            //  (likely BEFORE you run the function creating these, so you can use it as actual read times).
-            writeTime?: Time;
-            /** If you are writing values you KNOW you won't reading back, set this flag to
-            *      prevent caching of the writes clientside (which is a lot more efficient).
-            */
-            noWritePrediction?: boolean;
-            /** See PathValue.event */
-            eventWrite?: boolean;
-            /** Causes us to NOT ACTUALLY write values, but just return what we would write, if !dryRun */
-            dryRun?: boolean;
-        }
-    ): PathValue[] {
-        const { values, locks, eventPaths } = config;
-
-        // We trust our caller to use getFreeWriteTime, as they need to know the exact time
-        //  they are using so they can use it for their readTimes
-        let writeTime = config.writeTime;
-        if (!writeTime || writeTime.time === 0) {
-            writeTime = getNextTime();
-        }
-
-        let event = !!config.eventWrite;
-
-        let debugName = getOwnMachineId().slice(0, 8);
-        if (this.activeWatchSpec?.debugName) {
-            debugName += "|" + this.activeWatchSpec.debugName;
-        }
-
-        let pathValues: PathValue[] = [];
-        for (let [path, value] of values) {
-            let valueIsEvent = event;
-            if (eventPaths && eventPaths.has(path)) {
-                valueIsEvent = true;
-            }
-            let isTransparent = value === undefined || config.forcedUndefinedWrites?.has(path);
-            let pathValue: PathValue = pathValueCtor({
-                path,
-                valid: true,
-                value,
-                canGCValue: value === undefined,
-                time: writeTime,
-                locks: locks,
-                lockCount: locks.length,
-                event: valueIsEvent,
-                isTransparent,
-                source: debugName,
-            });
-            pathValues.push(pathValue);
-        }
-
-        if (!config.dryRun) {
-            pathValueCommitter.commitValues(pathValues, config.noWritePrediction ? undefined : "predictWrites");
-        }
-
-        return pathValues;
-    }
-
-    public unwatchEventPaths(paths: Set<string>) {
-        let relevantWatches = new Set<WatchSpec>();
-        for (let path of paths) {
-            let watchers = this.valueFunctionWatchers.get(path);
-            if (watchers) {
-                this.valueFunctionWatchers.delete(path);
-                for (let watcher of watchers.values()) {
-                    watcher.paths.delete(path);
-                    relevantWatches.add(watcher);
-                }
-            }
-            let basePath = hack_stripPackedPath(path);
-            let parentWatcher = this.parentValueFunctionWatchers.get(basePath);
-            if (parentWatcher) {
-                let watcherObj = parentWatcher.get(path);
-                if (watcherObj) {
-                    parentWatcher.delete(path);
-                    if (parentWatcher.size === 0) {
-                        this.parentValueFunctionWatchers.delete(basePath);
-                    }
-                    for (let watcher of watcherObj.lookup.values()) {
-                        watcher.parentPaths.delete(path);
-                        relevantWatches.add(watcher);
-                    }
-                }
-            }
-        }
-        for (let watcher of relevantWatches) {
-            watcher.unwatchEventPaths?.(paths);
-        }
-        remoteWatcher.unwatchEventPaths({ paths: Array.from(paths), parentPaths: Array.from(paths), });
-    }
-
-    public pathHasAnyWatchers(path: string) {
-        let parentPath = getParentPathStr(path);
-        return !!this.valueFunctionWatchers.get(path)?.size || !!this.parentValueFunctionWatchers.get(parentPath)?.size;
-    }
-    public getWatchersForPath(path: string) {
-        let matched = new Set<WatchSpec>();
-        let parentPath = getParentPathStr(path);
-        let watchers = this.valueFunctionWatchers.get(path);
-        if (watchers) {
-            for (let watcher of watchers.values()) {
-                matched.add(watcher);
-            }
-        }
-        let parentWatchers = this.parentValueFunctionWatchers.get(parentPath);
-        if (parentWatchers) {
-            for (let [_, watcherObj] of parentWatchers) {
-                for (let watcher of watcherObj.lookup.values()) {
-                    if (!matchesParentRangeFilter({ parentPath, fullPath: path, start: watcherObj.start, end: watcherObj.end })) {
-                        continue;
-                    }
-                    matched.add(watcher);
-                }
-            }
-        }
-        return matched;
-    }
-}
-export const clientWatcher = new ClientWatcher();
-(globalThis as any).clientWatcher = clientWatcher;
-(globalThis as any).ClientWatcher = ClientWatcher;
-
-void Promise.resolve().finally(() => {
-    authorityStorage.watchEventRemovals(x => clientWatcher.unwatchEventPaths(x));
-    pathWatcher.watchAllLocalTriggers((x, y) => clientWatcher.localOnValueCallback(x, y));
-});
-
-
+/**
+    Does everything needed to synchronize values with PathValueController, including
+        storing the values. While this can result in redundant storage, it helps keep
+        our strictly synchronized values, with values we just get from another node.
+        - We also don't consider rejections at all, and just assume there is no
+            contention in this file.
+        - Keys based on baseTime, so our values can be correctly clobbered by the remote.
+   
+        setValues
+            - For writing values
+        getValue
+            - For getting values
+        setWatches
+            - For knowing when values change
+ */
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { cache, lazy } from "socket-function/src/caching";
+import { binarySearchIndex, isNode, recursiveFreeze, sort } from "socket-function/src/misc";
+import { logErrors } from "../errors";
+import { getParentPathStr, getPathFromStr, hack_stripPackedPath } from "../path";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { pathValueCommitter, PathValueController } from "../0-path-value-core/PathValueController";
+import { PathValue, Value, getNextTime, Time, ReadLock, MAX_CHANGE_AGE, authorityStorage, getCreatorId, WatchConfig } from "../0-path-value-core/pathValueCore";
+import { pathWatcher } from "../0-path-value-core/PathWatcher";
+import { blue, green, red } from "socket-function/src/formatting/logColors";
+import { MaybePromise } from "socket-function/src/types";
+import { batchFunction, batchFunctionNone, runInfinitePoll } from "socket-function/src/batching";
+import { registerResource } from "../diagnostics/trackResources";
+import debugbreak from "debugbreak";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { PromiseObj } from "../promise";
+import { getOwnMachineId } from "../-a-auth/certs";
+import { remoteWatcher } from "./RemoteWatcher";
+import { heapTagObj } from "../diagnostics/heapTag";
+import { isDevDebugbreak } from "../config";
+import { auditLog } from "../0-path-value-core/auditLogs";
+import { decodeParentFilter, matchesParentRangeFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
+
+const pathValueCtor = heapTagObj("PathValue");
+
+export interface WatchSpecData {
+    paths: Set<string>;
+    pathSources: Set<PathValue> | undefined;
+    newParentsSynced: Set<string>;
+    extraReasons?: string[];
+}
+
+export interface WatchSpec {
+    debugName?: string;
+
+    // Run order is determined by the lowest [orderGroup, order]. It is HIGHLY recommended
+    //      to use orderGroup, with low values (0 is the default, so use 1 to run later,
+    //      and -1 to run earlier).
+    //      - Using a large number of unique orderGroups (for example, a unique group per callback),
+    //          will slow down callbacks.
+    //      - "order" is defaulted to a good default (the construct order), and so setting it will often
+    //          result in callbacks being given too high of a priority.
+    order?: number;
+    orderGroup?: number;
+
+    // callback is the key used to update watches, if the same callback is used
+    //  it won't start a new watch, but will instead update the paths used.
+    callback: (changed: WatchSpecData) => void;
+    unwatchEventPaths?: (paths: Set<string>) => void;
+    paths: Set<string>;
+    parentPaths: Set<string>;
+    /** For PathValueProxyWatcher. BUT, not in the Querysub routing service, or most backend middle-man type services. */
+    noInitialTrigger?: boolean;
+
+    // IMPORTANT! Any variables that should be persisted, must be copied over in updateUnwatches.
+    //  Otherwise we don't keep them. We do this for GC reasons (I think?).
+    //  - We might want to try reusing the same object... it should greatly reduce our GC overhead...
+    breakOnNextTrigger?: boolean;
+    triggerCount?: number;
+}
+
+let nextWatchSpecOrder = 1;
+export class ClientWatcher {
+    public static DEBUG_READS = false;
+    public static DEBUG_WRITES = false;
+    public static DEBUG_READS_EXPANDED = false;
+    public static DEBUG_WRITES_EXPANDED = false;
+    public static DEBUG_TRIGGERS?: "light" | "heavy";
+    public static DEBUG_SOURCES = false;
+
+    /** How long we keep watching, despite a value no longer being needed. */
+    public static WATCH_STICK_TIME = MAX_CHANGE_AGE * 2;
+    public static MAX_TRIGGER_TIME_BROWSER = 1000 * 5;
+    public static MAX_TRIGGER_TIME_NODEJS = 1000 * 300;
+
+
+    private valueFunctionWatchers = registerResource("paths|valueFunctionWatchers", new Map<string, Map<WatchSpec["callback"], WatchSpec>>());
+    // hack_stripPackedPath(path) =>
+    private parentValueFunctionWatchers = registerResource("paths|parentValueFunctionWatchers", new Map<string,
+        // path =>
+        Map<string, {
+            // start: number;
+            // end: number;
+            lookup: Map<WatchSpec["callback"], WatchSpec>
+        }>
+    >());
+    private allWatchers = registerResource("paths|clientWatcher.allWatchers", new Map<WatchSpec["callback"], WatchSpec>());
+
+    // path => expiryTime
+    private pendingUnwatches = registerResource("paths|pendingUnwatches", new Map<string, number>());
+    private pendingParentUnwatches = registerResource("paths|pendingParentUnwatches", new Map<string, number>());
+
+
+    private inLoop = false;
+    private isLoopSynchronous = false;
+    private pendingTriggered: Map<WatchSpec, WatchSpecData> | undefined;
+
+    private preTrigger(spec: WatchSpec) {
+        spec.triggerCount = (spec.triggerCount ?? 0) + 1;
+        if (spec.breakOnNextTrigger) {
+            spec.breakOnNextTrigger = false;
+            debugger;
+        }
+    }
+
+    private triggerWatcher(spec: WatchSpec, config?: { synchronous?: boolean }) {
+        this.preTrigger(spec);
+        let watchedTriggers = this.pendingTriggered = this.pendingTriggered || (new Map() as never);
+        let trigger = watchedTriggers.get(spec);
+        if (!trigger) {
+            trigger = {
+                pathSources: ClientWatcher.DEBUG_SOURCES ? new Set() : undefined,
+                newParentsSynced: new Set(),
+                paths: new Set(),
+            };
+            watchedTriggers.set(spec, trigger);
+        }
+        this.triggerWatchLoop(config);
+        return trigger;
+    }
+    private beforeTriggerDone: Promise<void> | undefined;
+    private onTriggerDone: Promise<void> | undefined;
+    public activeWatchSpec: WatchSpec | undefined;
+    private ignoreWatch: ((spec: WatchSpec, source: PathValue | string) => boolean) | undefined;
+    // Needed for typesafecss. Same as onTriggerDone, but fires first.
+    public waitForBeforeTriggerFinished() { return this.beforeTriggerDone; }
+    public waitForTriggerFinished() { return this.onTriggerDone; }
+    public isCurrentLoopSynchronous() { return this.isLoopSynchronous; }
+
+    @measureFnc
+    public localOnValueCallback(values: PathValue[], parentsSynced: Set<string>) {
+        if (values.length === 0 && parentsSynced.size === 0) return;
+        const onTrigger = (spec: WatchSpec, path: string, source?: PathValue) => {
+            if (
+                ClientWatcher.DEBUG_TRIGGERS === "heavy"
+                && !this.pendingTriggered?.has(spec)
+            ) {
+                let sourceName = "";
+                if (this.activeWatchSpec) {
+                    sourceName = ` (inside ${this.activeWatchSpec.debugName})`;
+                } else if (source?.source) {
+                    sourceName = ` (remote ${source.source.split("|").at(-1)})`;
+                }
+                console.log(`${blue("QUEUEING TRIGGER")} ${spec.debugName} ${sourceName}`);
+                console.log(`    DUE TO WRITE ${getPathFromStr(path).join(".")}`);
+            }
+        };
+
+        const triggerWatcher = (spec: WatchSpec, source: PathValue) => {
+            // Ignore self writes, otherwise local writes as simple as `x++`, will trigger an infinite loop.
+            if (this.ignoreWatch?.(spec, source)) return;
+            let trigger = this.triggerWatcher(spec);
+            if (trigger.pathSources) {
+                trigger.pathSources.add(source);
+            }
+            trigger.paths.add(source.path);
+            onTrigger(spec, source.path, source);
+        };
+        const triggerWatcherParent = (spec: WatchSpec, parent: string) => {
+            if (this.ignoreWatch?.(spec, parent)) return;
+            let trigger = this.triggerWatcher(spec);
+            trigger.newParentsSynced.add(parent);
+            onTrigger(spec, parent);
+        };
+        for (let value of values) {
+            let watchers = this.valueFunctionWatchers.get(value.path);
+            if (watchers) {
+                for (let watcher of watchers.values()) {
+                    triggerWatcher(watcher, value);
+                }
+            }
+            let parentPath = getParentPathStr(value.path);
+            let parentWatchers = this.parentValueFunctionWatchers.get(parentPath);
+            if (parentWatchers) {
+                // NOTE: We don't filter by path shard here
+                for (let [packedPath, { lookup }] of parentWatchers) {
+                    if (!matchesParentRangeFilter({ parentPath, fullPath: value.path, packedPath })) continue;
+                    for (let watcher of lookup.values()) {
+                        triggerWatcher(watcher, value);
+                        triggerWatcherParent(watcher, parentPath);
+                    }
+                }
+            }
+        }
+        // NOTE: We need to keep track of the parents being synced independently of the values, as you might request for all the child values, and there could be none. And so, this will be the only pathway which can actually trigger the watcher to rerun. And when it reruns, a previous call should have ingested the fact that the parent was synced to the authority storage. And so it should now decide that it's synced and stop waiting for the parent to sync values and commit (finish, render, etc, whatever it wants to do when it's fully synced).
+        for (let path of parentsSynced) {
+            let basePath = hack_stripPackedPath(path);
+            let watchers = this.parentValueFunctionWatchers.get(basePath);
+            if (watchers) {
+                let realObj = watchers.get(path);
+                if (realObj) {
+                    for (let watcher of realObj.lookup.values()) {
+                        triggerWatcherParent(watcher, path);
+                    }
+                }
+            }
+        }
+        // Trigger all children of parents synced
+        // NOTE: This is the counterpoint to RemoteWatcher's removal of extra requests when we are already syncing the parent path. If it's not going to request them, we're not going to get the paths, so we need to trigger based on parents.
+        //  - We ignore hack paths here (that path in range requests in parent paths), because RemoteWatcher also ignores them (so hacked parent paths won't stop value path syncs there), so they will be received normally.
+        // NOTE: This will probably be slow, but we shouldn't be receiving parent syncs very frequently. Or maybe we will, and maybe this will just be incredibly slow. 
+        if (parentsSynced.size > 0) {
+            measureBlock(() => {
+                for (let parentPath of parentsSynced) {
+                    for (let [path, lookup] of this.valueFunctionWatchers.entries()) {
+                        if (!path.startsWith(parentPath)) continue;
+                        if (getParentPathStr(path) !== parentPath) continue;
+                        for (let watch of lookup.values()) {
+                            let trigger = this.triggerWatcher(watch);
+                            trigger.paths.add(path);
+                            onTrigger(watch, path);
+                        }
+                    }
+                }
+            }, "ClientWatcher|triggerChildrenOfParentsSynced");
+        }
+    }
+
+    public debugBreakOnNextTrigger() {
+        if (!this.activeWatchSpec) throw new Error(`Break on next trigger must be called inside of a clientWatcher trigger`);
+        this.activeWatchSpec.breakOnNextTrigger = true;
+    }
+
+    public explicitlyTriggerWatcher(callback: WatchSpec["callback"], config?: { synchronous?: boolean }) {
+        let watcher = this.allWatchers.get(callback);
+        if (!watcher) throw new Error(`No watcher found for callback ${callback.toString()}`);
+        this.triggerWatcher(watcher, config);
+    }
+
+    private triggerWatchLoop(config?: { synchronous?: boolean }) {
+        // A fairly standard trigger loop. Batch synchronous triggers for efficiency's sake
+        if (this.inLoop) return;
+        this.inLoop = true;
+        let beforeTriggerDone = new PromiseObj();
+        let triggerPromiseObj = new PromiseObj();
+        this.beforeTriggerDone = beforeTriggerDone.promise;
+        this.onTriggerDone = triggerPromiseObj.promise;
+        const doLoop = () => {
+            try {
+                this.innerTriggerLoop();
+            } finally {
+                this.inLoop = false;
+                this.isLoopSynchronous = false;
+                beforeTriggerDone.resolve();
+                triggerPromiseObj.resolve();
+                this.beforeTriggerDone = undefined;
+                this.onTriggerDone = undefined;
+            }
+        };
+        if (config?.synchronous) {
+            this.isLoopSynchronous = true;
+            doLoop();
+        } else {
+            void Promise.resolve().finally(doLoop);
+        }
+    }
+    @measureFnc
+    private innerTriggerLoop() {
+        let loopCount = 0;
+        let triggerCount = 0;
+        let groupStart = (
+            ClientWatcher.DEBUG_TRIGGERS === "light" && console.groupCollapsed
+            || ClientWatcher.DEBUG_TRIGGERS === "heavy" && console.group
+            || undefined
+        );
+
+        for (let callback of this.loopStartCallbacks) {
+            try {
+                callback();
+            } catch (e) {
+                logErrors(e);
+            }
+        }
+        let MAX_TRIGGER_TIME = isNode() ? ClientWatcher.MAX_TRIGGER_TIME_NODEJS : ClientWatcher.MAX_TRIGGER_TIME_BROWSER;
+
+        let time = Date.now();
+        while (true) {
+            let currentTriggered = this.pendingTriggered;
+            this.pendingTriggered = undefined;
+            if (!currentTriggered?.size) break;
+
+            if (ClientWatcher.DEBUG_TRIGGERS) {
+                console.log(" ");
+            }
+            groupStart?.(`${blue("TRIGGERS LOOP ITERATION")} ${loopCount++}`);
+
+            let sorted = Array.from(currentTriggered);
+            sorted.sort((a, b) => {
+                let orderDiff = (a[0].orderGroup ?? 0) - (b[0].orderGroup ?? 0);
+                if (orderDiff !== 0) return orderDiff;
+                return (a[0].order ?? 0) - (b[0].order ?? 0);
+            });
+            if (ClientWatcher.DEBUG_TRIGGERS) {
+                for (let [trigger] of sorted) {
+                    console.log("    " + trigger.debugName + " " + (trigger.orderGroup ?? 0) + "|" + (trigger.order ?? 0));
+                }
+                console.groupEnd();
+            }
+            let curIndex = 0;
+            this.ignoreWatch = (spec, source) => {
+                this.preTrigger(spec);
+                // Ignore it if we are handling it (self triggers are almost always just due to `x++`, and are virtually
+                //  never intended to actually self trigger. And if they are, then they can just detach with
+                //  Promise.resolve.finally(...))
+                // ALSO ignore if we are going to handle it. There's no reason to queue another watch for something
+                //  we are about to handle.
+                for (let checkIndex = curIndex; checkIndex < sorted.length; checkIndex++) {
+                    if (sorted[checkIndex][0] === spec) {
+                        let watchSpecData = sorted[checkIndex][1];
+                        if (typeof source === "string") {
+                            watchSpecData.newParentsSynced.add(source);
+                        } else {
+                            if (watchSpecData.pathSources) {
+                                watchSpecData.pathSources.add(source);
+                            }
+                            watchSpecData.paths.add(source.path);
+                        }
+                        return true;
+                    }
+                }
+
+                // If they care about the order, and it is higher than the current order, then insert it into the current
+                if (spec.order !== undefined && (
+                    (spec.order ?? 0) > (sorted[curIndex][0].order ?? 0)
+                    && (spec.orderGroup ?? 0) === (sorted[curIndex][0].orderGroup ?? 0)
+                    || (spec.orderGroup ?? 0) > (sorted[curIndex][0].orderGroup ?? 0)
+                )) {
+                    if (ClientWatcher.DEBUG_TRIGGERS === "heavy") {
+                        console.log("   (insert new trigger)");
+                        console.log("    " + spec.debugName + " " + (spec.orderGroup ?? 0) + "|" + (spec.order ?? 0));
+                    }
+                    // Find the last insert >= the current order (check for equals so we can insert at the end
+                    //      duplicate order values).
+                    let insertIndex = sorted.findLastIndex(x =>
+                        (spec.order ?? 0) >= (x[0].order ?? 0)
+                        && (spec.orderGroup ?? 0) === (x[0].orderGroup ?? 0)
+                        || (spec.orderGroup ?? 0) > (x[0].orderGroup ?? 0)
+                    );
+                    if (insertIndex === -1) {
+                        insertIndex = sorted.length - 1;
+                    }
+                    insertIndex++;
+                    let watchSpecData: WatchSpecData = {
+                        paths: new Set(),
+                        pathSources: ClientWatcher.DEBUG_SOURCES ? new Set() : undefined,
+                        newParentsSynced: new Set(),
+                    };
+                    if (typeof source === "string") {
+                        watchSpecData.newParentsSynced.add(source);
+                    } else {
+                        if (watchSpecData.pathSources) {
+                            watchSpecData.pathSources.add(source);
+                        }
+                        watchSpecData.paths.add(source.path);
+                    }
+                    sorted.splice(insertIndex, 0, [spec, watchSpecData]);
+                    return true;
+                }
+                return false;
+            };
+
+            let stoppedEarly = false;
+            for (; curIndex < sorted.length; curIndex++) {
+                let [watchSpec, data] = sorted[curIndex];
+                triggerCount++;
+                if (ClientWatcher.DEBUG_TRIGGERS === "heavy") {
+                    console.log(`${green("RUNNING TRIGGER")} ${watchSpec.debugName}`);
+                }
+
+                try {
+                    this.activeWatchSpec = watchSpec;
+                    watchSpec.callback(data);
+                } catch (e) {
+                    void Promise.resolve().finally(() => { throw e; });
+                } finally {
+                    this.activeWatchSpec = undefined;
+                }
+                if (
+                    Date.now() - time > MAX_TRIGGER_TIME
+                    && !isDevDebugbreak()
+                ) {
+                    stoppedEarly = true;
+                    break;
+                }
+            }
+            this.ignoreWatch = undefined;
+
+            if (ClientWatcher.DEBUG_TRIGGERS) {
+                console.log(" ");
+            }
+
+            if (stoppedEarly) {
+                const history = 10;
+                let remaining = (
+                    sorted
+                        .slice(curIndex, curIndex + history)
+                        .map(x => `${x[0].debugName} (${Array.from(x[1].paths).map(x => getPathFromStr(x).join(".")).join(" | ")})`)
+                        .join(", ")
+                );
+                console.error(red(`Too much time spent in trigger loop, aborting loop. Triggered ${triggerCount} triggers. Remaining triggers: ${remaining}`), triggerCount);
+                break;
+            }
+        }
+
+        let finalStats = { loopCount, triggerCount };
+        for (let callback of this.loopEndCallbacks) {
+            try {
+                callback(finalStats);
+            } catch (e) {
+                logErrors(e);
+            }
+        }
+
+        time = Date.now() - time;
+        if (ClientWatcher.DEBUG_TRIGGERS) {
+            console.log(`${green("TRIGGERS LOOP FINISHED")} ${loopCount} loops, ${triggerCount} triggers, in ${time}ms`);
+            console.log(" ");
+            console.log(" ");
+        }
+    }
+
+    private loopStartCallbacks = new Set<() => void>();
+    private loopEndCallbacks = new Set<(info: { loopCount: number; triggerCount: number; }) => void>();
+    public onLoopStart(callback: () => void) {
+        this.loopStartCallbacks.add(callback);
+    }
+    public onLoopEnd(
+        callback: (info: {
+            loopCount: number;
+            triggerCount: number;
+        }) => void
+    ) {
+        this.loopEndCallbacks.add(callback);
+    }
+
+    public unwatch(callback: WatchSpec["callback"]) {
+        this.updateUnwatches({
+            callback,
+            paths: new Set(),
+            parentPaths: new Set(),
+        });
+        this.allWatchers.delete(callback);
+    }
+
+    @measureFnc
+    private updateUnwatches(watchSpec: WatchSpec) {
+        const { callback, paths, parentPaths } = watchSpec;
+        let prevSpec = this.allWatchers.get(callback);
+        if (!prevSpec) return;
+        if (prevSpec) {
+            watchSpec.order = prevSpec.order;
+            watchSpec.triggerCount = prevSpec.triggerCount;
+            watchSpec.breakOnNextTrigger = prevSpec.breakOnNextTrigger;
+        }
+
+        let fullyUnwatchedPaths = new Set<string>();
+        let fullyUnwatchedParents = new Set<string>();
+
+        // Remove removed paths
+        for (let path of prevSpec.paths) {
+            if (paths.has(path)) continue;
+            let watchers = this.valueFunctionWatchers.get(path);
+            if (!watchers) continue;
+            watchers.delete(callback);
+            if (watchers.size === 0) {
+                fullyUnwatchedPaths.add(path);
+                this.valueFunctionWatchers.delete(path);
+            }
+        }
+
+        // Remove removed parents
+        for (let path of prevSpec.parentPaths) {
+            if (parentPaths.has(path)) continue;
+            let basePath = hack_stripPackedPath(path);
+            let watchersBase = this.parentValueFunctionWatchers.get(basePath);
+            if (!watchersBase) continue;
+            let watchers = watchersBase.get(path);
+            if (!watchers) continue;
+            watchers.lookup.delete(callback);
+            if (watchers.lookup.size === 0) {
+                watchersBase.delete(path);
+                fullyUnwatchedParents.add(path);
+            }
+            if (watchersBase.size === 0) {
+                this.parentValueFunctionWatchers.delete(basePath);
+            }
+        }
+
+        prevSpec.paths = paths;
+        prevSpec.parentPaths = parentPaths;
+        watchSpec = prevSpec;
+
+        let expiryTime = Date.now() + ClientWatcher.WATCH_STICK_TIME;
+        // Commented out, as this is showing up as a bit slow in profiling
+        // if (isDevDebugbreak()) {
+        //     for (let path of fullyUnwatchedPaths) {
+        //         auditLog("clientWatcher UNWATCH", { path });
+        //     }
+        //     for (let path of fullyUnwatchedParents) {
+        //         auditLog("clientWatcher UNWATCH PARENT", { path });
+        //     }
+        // }
+        for (let path of fullyUnwatchedPaths) {
+            this.pendingUnwatches.set(path, expiryTime);
+        }
+        for (let path of fullyUnwatchedParents) {
+            this.pendingParentUnwatches.set(path, expiryTime);
+        }
+        this.ensureUnwatching();
+    }
+    private ensureUnwatching = lazy(() => {
+        let waitTime = Math.max(1000 * 5, ClientWatcher.WATCH_STICK_TIME - 1000);
+        const self = this;
+        runInfinitePoll(waitTime, async function ensureUnwatching() {
+            let now = Date.now();
+            let pathsToUnwatch = new Set<string>();
+            let parentPathsToUnwatch = new Set<string>();
+            for (let [path, expiryTime] of self.pendingUnwatches) {
+                if (expiryTime < now) {
+                    self.pendingUnwatches.delete(path);
+                    pathsToUnwatch.add(path);
+                }
+            }
+            for (let [path, expiryTime] of self.pendingParentUnwatches) {
+                if (expiryTime < now) {
+                    self.pendingParentUnwatches.delete(path);
+                    parentPathsToUnwatch.add(path);
+                }
+            }
+            if (pathsToUnwatch.size > 0 || parentPathsToUnwatch.size > 0) {
+                //console.log(red("Unwatching paths"), pathsToUnwatch, parentPathsToUnwatch, Date.now());
+                let unwatchConfig: WatchConfig = { paths: Array.from(pathsToUnwatch), parentPaths: Array.from(parentPathsToUnwatch), };
+                // pathWatcher unwatches remotely as well
+                pathWatcher.unwatchPath({ callback: getOwnNodeId(), ...unwatchConfig });
+            }
+        });
+    });
+
+
+    // TODO: EVENTUALLY! We should accept a delta on the watches, which should allow for more efficient updates.
+
+    // NOTE: The callback is only called when synced values change (so if they are already synced, and don't change, the callback won't be called).
+    // NOTE: Doesn't call the callback until all requested values have finished their initial sync, to prevent too many callback callbacks.
+    //  - You can just fragment your writes into different components to isolate any slow loading parts, so partial
+    //      loading of data really isn't needed.
+    // NOTE: Takes ownership of paths and parentPaths, so... don't mutate them after calling this!
+    @measureFnc
+    public setWatches(watchSpec: WatchSpec) {
+        // NOTE: Yes, setWatches shows up as being slow in the profiler. I added measureBlock to every single section, and none of them showed up as being slow.
+        //  - My best guess is that after we clobber values in various lookups, but BEFORE we return, the destructor for those old resources runs. So their destruct time gets put in our profiler?
+        //      - Or maybe hack_stripPackedPath, decodeParentFilter, or other functions with heavy temporary allocations.
+        //  - measureBlock also has overhead (which is why I commented them all out), but... It's not nearly enough to account for this.
+
+
+        //measureBlock(() => {
+        watchSpec.order = watchSpec.order ?? nextWatchSpecOrder++;
+        //}, "ClientWatcher()|watchSpec.order = ...");
+
+        //measureBlock(() => {
+        this.updateUnwatches(watchSpec);
+        //}, "ClientWatcher()|updateUnwatches wrapper?");
+
+        //measureBlock(() => {
+        this.allWatchers.set(watchSpec.callback, watchSpec);
+        //}, "ClientWatcher()|setWatches|this.allWatcher.set");
+        //measureBlock(() => {
+        for (let path of watchSpec.paths) {
+            let watchers = this.valueFunctionWatchers.get(path);
+            if (!watchers) {
+                watchers = new Map();
+                this.valueFunctionWatchers.set(path, watchers);
+            }
+            watchers.set(watchSpec.callback, watchSpec);
+            this.pendingUnwatches.delete(path);
+        }
+        //}, "ClientWatcher()|setWatches|paths");
+        //measureBlock(() => {
+        for (let path of watchSpec.parentPaths) {
+            let basePath = hack_stripPackedPath(path);
+            let watchersBase = this.parentValueFunctionWatchers.get(basePath);
+            if (!watchersBase) {
+                watchersBase = new Map();
+                this.parentValueFunctionWatchers.set(basePath, watchersBase);
+            }
+            let watchers = watchersBase.get(path);
+            if (!watchers) {
+                watchers = {
+                    lookup: new Map()
+                };
+                watchersBase.set(path, watchers);
+            }
+            watchers.lookup.set(watchSpec.callback, watchSpec);
+            this.pendingParentUnwatches.delete(path);
+        }
+        //}, "ClientWatcher()|setWatches|parentPaths");
+
+        if (watchSpec.paths.size === 0 && watchSpec.parentPaths.size === 0) {
+            return;
+        }
+
+        //measureBlock(() => {
+        let pathsArray = Array.from(watchSpec.paths);
+        let parentPathsArray = Array.from(watchSpec.parentPaths);
+        let debugName = watchSpec.callback.name;
+        pathWatcher.watchPath({
+            nodeId: getOwnNodeId(),
+            paths: pathsArray,
+            parentPaths: parentPathsArray,
+            debugName,
+            noInitialTrigger: watchSpec.noInitialTrigger,
+        });
+        //}, "ClientWatcher()|setWatches|watchCalls");
+    }
+
+    private lastVersions = registerResource("paths|lastVersion", new Map<number, number>());
+    private getFreeVersionBase(time: number): number {
+        this.ensureCleaningUpVersions();
+        let lastVersion = this.lastVersions.get(time);
+        let nextVersion = lastVersion === undefined ? 0 : lastVersion + 1;
+        this.lastVersions.set(time, nextVersion);
+        return nextVersion;
+    }
+    private ensureCleaningUpVersions = lazy(() => {
+        const self = this;
+        const THRESHOLD = MAX_CHANGE_AGE * 2;
+        runInfinitePoll(THRESHOLD, function cleanUpVersions() {
+            let curThreshold = Date.now() - THRESHOLD;
+            for (let time of self.lastVersions.keys()) {
+                if (time < curThreshold) {
+                    self.lastVersions.delete(time);
+                }
+            }
+        });
+    });
+
+    /** Gets a free write time by incrementing the version, and then ensuring that version isn't reused for the given time. */
+    public getFreeWriteTime(time: Time | undefined): Time {
+        if (!time?.time) return getNextTime();
+        let version = this.getFreeVersionBase(time.time);
+        return { time: time.time, version, creatorId: getCreatorId() };
+    }
+
+    // IMPORTANT! The entire path from setValues => localOnValueCallback MUST be synchronous. If it is asynchronous we CANNOT
+    //      correctly prevent watchers from triggering themselves, which can result in either 2X the number of needed evaluations,
+    //      OR infinite evaluations.
+    // NOTE: This should be called RIGHT after you read the values, otherwise the lock that is created
+    //  may not depend on what was actually read.
+    // NOTE: If a server goes down, these writes may fail to commit (they will depend on each other, so
+    //  it will be either all or nothing).
+    // NOTE: This synchronously predicts the values, so you don't need to wait. When the promise resolves
+    //  the values will be committed and they won't belost (although it is still possible for them to become
+    //  rejected).
+    @measureFnc
+    public setValues(
+        config: {
+            values: Map<string, Value>;
+            forcedUndefinedWrites?: Set<string>;
+            eventPaths?: Set<string>;
+            locks: ReadLock[];
+            // NOTE: Use clientWatcher.getFreeWriteTime() to ensure this writeTime is valid
+            //  (likely BEFORE you run the function creating these, so you can use it as actual read times).
+            writeTime?: Time;
+            /** If you are writing values you KNOW you won't reading back, set this flag to
+            *      prevent caching of the writes clientside (which is a lot more efficient).
+            */
+            noWritePrediction?: boolean;
+            /** See PathValue.event */
+            eventWrite?: boolean;
+            /** Causes us to NOT ACTUALLY write values, but just return what we would write, if !dryRun */
+            dryRun?: boolean;
+            source?: string;
+        }
+    ): PathValue[] {
+        const { values, locks, eventPaths } = config;
+
+        // We trust our caller to use getFreeWriteTime, as they need to know the exact time
+        //  they are using so they can use it for their readTimes
+        let writeTime = config.writeTime;
+        if (!writeTime || writeTime.time === 0) {
+            writeTime = getNextTime();
+        }
+
+        let event = !!config.eventWrite;
+
+        let source = config.source;
+
+        if (!source) {
+            let debugName = getOwnMachineId().slice(0, 8);
+            if (this.activeWatchSpec?.debugName) {
+                debugName += "|" + this.activeWatchSpec.debugName;
+            }
+            source = debugName;
+        }
+
+        let pathValues: PathValue[] = [];
+        for (let [path, value] of values) {
+            let valueIsEvent = event;
+            if (eventPaths && eventPaths.has(path)) {
+                valueIsEvent = true;
+            }
+            let isTransparent = value === undefined || config.forcedUndefinedWrites?.has(path);
+            let pathValue: PathValue = pathValueCtor({
+                path,
+                valid: true,
+                value,
+                canGCValue: value === undefined,
+                time: writeTime,
+                locks: locks,
+                lockCount: locks.length,
+                event: valueIsEvent,
+                isTransparent,
+                source,
+            });
+            pathValues.push(pathValue);
+        }
+
+        if (!config.dryRun) {
+            pathValueCommitter.commitValues(pathValues, config.noWritePrediction ? undefined : "predictWrites");
+        }
+
+        return pathValues;
+    }
+
+    public unwatchEventPaths(paths: Set<string>) {
+        let relevantWatches = new Set<WatchSpec>();
+        for (let path of paths) {
+            let watchers = this.valueFunctionWatchers.get(path);
+            if (watchers) {
+                this.valueFunctionWatchers.delete(path);
+                for (let watcher of watchers.values()) {
+                    watcher.paths.delete(path);
+                    relevantWatches.add(watcher);
+                }
+            }
+            let basePath = hack_stripPackedPath(path);
+            let parentWatcher = this.parentValueFunctionWatchers.get(basePath);
+            if (parentWatcher) {
+                let watcherObj = parentWatcher.get(path);
+                if (watcherObj) {
+                    parentWatcher.delete(path);
+                    if (parentWatcher.size === 0) {
+                        this.parentValueFunctionWatchers.delete(basePath);
+                    }
+                    for (let watcher of watcherObj.lookup.values()) {
+                        watcher.parentPaths.delete(path);
+                        relevantWatches.add(watcher);
+                    }
+                }
+            }
+        }
+        for (let watcher of relevantWatches) {
+            watcher.unwatchEventPaths?.(paths);
+        }
+        remoteWatcher.unwatchEventPaths({ paths: Array.from(paths), parentPaths: Array.from(paths), });
+    }
+
+    public pathHasAnyWatchers(path: string) {
+        let parentPath = getParentPathStr(path);
+        return !!this.valueFunctionWatchers.get(path)?.size || !!this.parentValueFunctionWatchers.get(parentPath)?.size;
+    }
+    public getWatchersForPath(path: string) {
+        let matched = new Set<WatchSpec>();
+        let parentPath = getParentPathStr(path);
+        let watchers = this.valueFunctionWatchers.get(path);
+        if (watchers) {
+            for (let watcher of watchers.values()) {
+                matched.add(watcher);
+            }
+        }
+        let parentWatchers = this.parentValueFunctionWatchers.get(parentPath);
+        if (parentWatchers) {
+            for (let [packedPath, watcherObj] of parentWatchers) {
+                for (let watcher of watcherObj.lookup.values()) {
+                    if (!matchesParentRangeFilter({ parentPath, fullPath: path, packedPath })) continue;
+                    matched.add(watcher);
+                }
+            }
+        }
+        return matched;
+    }
+}
+export const clientWatcher = new ClientWatcher();
+(globalThis as any).clientWatcher = clientWatcher;
+(globalThis as any).ClientWatcher = ClientWatcher;
+
+void Promise.resolve().finally(() => {
+    authorityStorage.watchEventRemovals(x => clientWatcher.unwatchEventPaths(x));
+    pathWatcher.setLocalTriggerCallback((x, y) => clientWatcher.localOnValueCallback(x, y));
+});
+
+