querysub 0.2.0

package/src/0-path-value-core/pathValueCore.ts

@@ -0,0 +1,2217 @@
+import debugbreak from "debugbreak";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { cache, cacheArgsEqual, lazy } from "socket-function/src/caching";
+import { addEpsilons, getBufferFraction, minusEpsilon } from "../bits";
+import { errorToUndefined, ignoreErrors, logErrors } from "../errors";
+import { appendToPathStr, getParentPathStr, getPathDepth, getPathFromStr, getPathIndexAssert, getPathStr, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath, removePathLastPart, rootPathStr } from "../path";
+import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { IdentityController_getOwnPubKeyShort, debugNodeId } from "../-c-identity/IdentityController";
+import { pathValueArchives } from "./pathValueArchives";
+import { blue, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { PathValueController, pathValueCommitter } from "./PathValueController";
+import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { markArrayAsSplitable } from "socket-function/src/fixLargeNetworkCalls";
+import { registerDynamicResource, registerMapArrayResource, registerResource } from "../diagnostics/trackResources";
+import { binarySearchIndex, isNode, isNodeTrue, promiseObj, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { isNodeTrusted, isTrusted, isTrustedByNode } from "../-d-trust/NetworkTrust2";
+import { AuthorityPath, LOCAL_DOMAIN_PATH, pathValueAuthority2 } from "./NodePathAuthorities";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { isNoNetwork } from "../config";
+import { formatTime } from "socket-function/src/formatting/format";
+import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getNodeIdDomain, getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
+import { getMachineId } from "../-a-auth/certs";
+import { PromiseRace } from "../../src/misc/PromiseRace";
+
+import yargs from "yargs";
+import { sha256 } from "js-sha256";
+import { PromiseObj } from "../promise";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { debugLog, isDebugLogEnabled } from "./debugLogs";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("noarchive", { type: "boolean", alias: ["noarchives"], desc: "Don't save writes to disk. Still reads from disk." })
+    .argv || {}
+    ;
+
+/** We don't ingest changes older than this. However, once changes are ingested,
+ *      we don't usually verify them, and if we did, it would be against a higher value
+ *      (but NOT MAX_CHANGE_AGE, as that is too high).
+ */
+export const MAX_ACCEPTED_CHANGE_AGE = 1000 * 30;
+/** We reject changes older than this. */
+//export let MAX_ACCEPTED_CHANGE_AGE = 1000 * 60 * 2;
+
+/** After this amount of time, we assume no new changes will be created
+ *      (which means changes will no longer become rejected)
+ *      (Should be a bit longer than MAX_ACCEPTED_CHANGE_AGE, as there might be a delay
+ *          in processing changes, so it might be past MAX_ACCEPTED_CHANGE_AGE, but we might
+ *          still receive changes in the pipeline).
+ *  NOTE: This is now MUCH higher than MAX_ACCEPTED_CHANGE_AGE, because of the case of the fast writer and slow reader.
+ *      The slow reader can easily never get the read finished before the writer updates it. We mask the issue by moving
+ *      the slow reader farther in the past, which helps the case where it is only reading, although causes some instability
+ *      if the function also writes to values (as it can cause rejections that go back as far into the past as it is writing).
+ *      Of course this won't always work, but it should solve most cases (the true solution is just running those functions
+ *      on the same thread, but then we take a massive speed hit, and the fast writer starts to lag).
+ *      - MAX_ACCEPTED_CHANGE_AGE should probably become even shorter, and maybe even dynamic, as in games where
+ *          you see the rejections going back 60s would be really annoying.
+ *  NOTE: Any values before age time are subject to being received out of order. AFTER this time,
+ *      every authority should have all values, and therefore, all reads should be able to see
+ *      data in the correct (global) order.
+ */
+export const MAX_CHANGE_AGE = MAX_ACCEPTED_CHANGE_AGE * 2;
+
+/** Extra time we keep clientside prediction rejections for, to give us time to receive the actual values. */
+export const CLIENTSIDE_PREDICT_LEEWAY = 500;
+
+/** Any PathValues which take longer than this to wrist should be rejected, so... we have
+ *      to write well before this time.
+ *      - This has to be at least MAX_CHANGE_AGE * 4.5 + the time to serialize and
+ *          send our data to remote storage.
+ */
+export const ARCHIVE_FLUSH_LIMIT = Math.max(MAX_CHANGE_AGE * 10, timeInMinute * 60);
+
+/** After this we start considering reading values and GCing them. Before this we
+ *      just merge the values, not really processing them.
+ */
+export const VALUE_GC_THRESHOLD = ARCHIVE_FLUSH_LIMIT * 3;
+
+// Only start using ARCHIVE_FLUSH_LIMIT after this time.
+//  Before this date, we moves some files around, so their timestamps are too old.
+//      Once we upload to backblaze we MIGHT have to reset this, but... really, any future moves
+//      should change the type (the limit is only for 'genesis' type files).
+export const ARCHIVE_FLUSH_LIMIT_START = +new Date("2024-07-14");
+
+/** The maximum time we assume a completed archive write will take to be visible to any reader.
+ *      - We also assume any files with file.writeTime + MAX_ARCHIVE_PROPAGATION_DELAY < Date.now()
+ *          will be visible to all readers.
+ */
+export const MAX_ARCHIVE_PROPAGATION_DELAY = 1000 * 10;
+
+/** The time we wait until we cleanup undefined values from memory. */
+export const UNDEFINED_MEMORY_CLEANUP_DELAY = MAX_CHANGE_AGE * 2;
+
+export const ARCHIVE_LOOP_DELAY = MAX_CHANGE_AGE * 2;
+
+/** Both of these are just targets, we might exceed them. */
+export const FILE_VALUE_COUNT_LIMIT = 50_000;
+export const FILE_SIZE_LIMIT = 50_000_000;
+
+export const THREAD_START_TIME = Date.now();
+
+/** On startup we are more lenient waiting for things, in case many services are started
+ *      at the same time. But... eventually, things need to be loaded, and we have to start
+ *      throwing errors
+ */
+export const STARTUP_CUTOFF_TIME = THREAD_START_TIME + timeInSecond * 30;
+
+/** After this time, we assume an an authority is dead, and stop waiting for it to startup.
+ *      - Most of this time is spent loading the database, which can easily take minutes.
+ *      - Authority startups should be rare, so this can be somewhat long.
+ *      - If are smart about ignoring nodes which are not responding to commands,
+ *          so it should be rare for zombie nodes to exist, but... it could happen.
+*/
+export const MAX_ACCEPTED_AUTHORITY_STARTUP_TIME = timeInMinute * 30;
+
+
+// NOTE: `time` maybe a bit off, as it must be unique, so if we create an identical value we
+//  will increment it by epsilon (addEpsilons). This gives us 4096 values per millisecond before
+//  we start to drift, which... is way more than enough (4M values per second would be a LOT of writes).
+//  `creatorId` is the first 62 bits (can't use 64 as !== with double is not the same as uint64)
+//      of the public key of the caller. This can be validated (by IdentityController right now),
+//      but as only trusted nodes create writes we don't really need to validate it. There is a new
+//      creatorId per process, which is few enough so that we shouldn't ever have to worry about
+//      collisions.
+// NOTE: Time is unique for each set of changes, and therefore for each path, BUT, it is NOT
+//  unique across paths (which should be fine, as different paths should never experience contention).
+//  - Values with === Time should also have === readLocks?
+export type Time = {
+    time: number;
+    // NOTE: Version is required, so that FunctionRunner can update a value without
+    //  changing the time order by too much. In most cases it will be 0.
+    // IMPORTANT! version should be an integer (it can be negative, just no decimals).
+    version: number;
+    creatorId: number;
+};
+export const epochTime: Time = { time: 0, version: 0, creatorId: 0 };
+Object.freeze(epochTime);
+
+/** `timeMinusEpsilon(time)` < time BUT, there is never a case where
+ *      `timeMinusEpsilon(time) < otherTime < time` (there is no gap between the times)
+ */
+export function timeMinusEpsilon(time: Time) {
+    return { time: time.time, version: time.version, creatorId: minusEpsilon(time.creatorId) };
+}
+
+function validateLockCount(value: PathValue) {
+    if (value.lockCount === 0 && value.locks.length > 0) {
+        throw new Error(`lockCount is 0, but locks are present (there are ${value.locks.length}), the value is invalid, lockCount MUST be set correctly`);
+    }
+}
+
+// The startTime MUST have a valid value. For `startTime < other.time < endTime`
+//  them there CANNOT be a valid value (if there is, it means there was contention,
+//  and the lock is rejected, invalidating the write that uses it).
+//  NOTE: endTime might NOT equal PathValue.time, due to reusing readLocks for the same
+//      write (but NOT reusing time, because time must be unique).
+// 'Locks' the path for the duration of the startTime/endTime. startTime is the time
+//      the value read was written at, which is required as we don't provide a value
+//      for the ReadLock, so we need a way to determine which value we are locking
+//      (and if it exists!).
+//      - The fact that the duration of the lock is longer than required is fine. We are
+//          extending it from the read time to the write time, which is usually far in the past.
+//          And the only way the extended duration would cause the lock to break, is if
+//          the value change away and the changes back again... which is a rare care anyways!
+export type ReadLock = {
+    path: string;
+    // Exclusive (for ranges), often === the time of the value read
+    // === (for single values), often === the time of the value read, but can be any value,
+    //  as long as it exists. UNLESS readIsTransparent, then it it must either not exist,
+    //  or be a transparent write.
+    startTime: Time;
+    // Exclusive (for ranges), often === the time we are writing at
+    endTime: Time;
+    // We don't store the ENTIRE value because that would result in sending the entire
+    //  program state per write (although I guess if you only read numbers ReadLock
+    //  is pretty big anyways, but... eh...). HOWEVER, sending if we are reading
+    //  undefined is useful, and makes garbage collection possible.
+    readIsTranparent: boolean;
+
+    /** A clientside only value to cause rejected values to be used as valid values for a bit, to fix
+     *      the case where we receive the function evaluation before the replacement values (which is
+     *      actually fairly common, due to how permissions work, so that existing paths are faster,
+     *      but new paths are slower).
+      */
+    keepRejectedUntil?: number;
+};
+// NOTE: WriteState is a pared down PathValue for ReadLocks to consume.
+//  1) This helps keep them separate, as values you read are very different from values
+//          you watch or are an authority on.
+//  2) This greatly reduces data sent over the wire.
+//  3) WriteStates can be thrown away after a period of time, as we can assume values are valid
+//      once they are old enough (as if they are old and invalid no one would be using them, and
+//      everyone would know to stop using them a long time ago). PathValues cannot, as they store
+//      data, which cannot be inferred, but as WriteStates don't store data, and only matter
+//      if they are inValid (or cause another value to be invalid), it works fine...
+export type WriteState = {
+    path: string;
+    time: Time;
+    isValid: boolean;
+    // We don't store the ENTIRE value, because that would be inefficient. But... to make
+    //  garbage collection possible, we store if it is `undefined`.
+    //  NOTE: We could also start to store simply values here, such as numbers. This would allow
+    //      dumb writers that don't want to check if their value exists to write without breaking
+    //      reads (ex, multiple might write database.name = "example" at once).
+    //      - HOWEVER, writers should just be smart, as it causes a lot of thrashing and work
+    //          on all the clients per write, even if nothing changes!
+    isTransparent: boolean;
+};
+
+// NOTE: Object values ARE allowed here. They are effectively immutable though, so it is recommended
+//  clients recursively freeze them after they deserialize them.
+export type Value = unknown;
+export type PathValue = {
+    path: string;
+    /** @deprecated ALWAYS access value using pathValueSerializer.getPathValue(), unless making a shallow copy. */
+    value: Value;
+    time: Time;
+    /** @deprecated NOT deprecated, just remember when you set this ALWAYS set lockCount, otherwise
+     *      valid checking breaks!
+     *  IMPORTANT! DO NOT check for locks.length === 0. Locks are lost during serialization, so this
+     *      will be irrelevant. lockCount on the other hand, should be accurate.
+     */
+    locks: ReadLock[];
+
+    /** Should be set EVERY time we have locks. IF lockCount === 0, then we know there are no locks
+     *      (this is preserved through serialization, even when locks aren't).
+     */
+    lockCount: number;
+
+    /** value === undefined
+     *      - MUST be set if value === undefined
+     *      - We only GC values if they aren't eclipsing any values (or if we also remove the values they are eclipsing
+     *          at the same time, in an atomic manner).
+     *      - Makes reading more efficient (so we don't have to worry about lazily reading values), and makes our
+     *          intentions more clear.
+     */
+    canGCValue?: boolean;
+
+    /** Event paths are temporary paths which are never used after a certain period of time
+     *      (NOT paths which are written to frequently, those should be normal writes).
+     *  Characteristics of event paths:
+     *      1) Not stored on disk
+     *      2) Not readable after `time.time + MAX_CHANGE_AGE`
+     *      3) Dropped from storage and watches after time: `time.time + MAX_CHANGE_AGE`
+     *  NOTE: This is per path, not per value. This means events MUST have unique paths,
+     *      as newer values will be deleted based on when the first use of the path occurred.
+     *  NOTE: Non-recursive, so it has to be on the exact paths
+     */
+    event?: boolean;
+
+    /*
+        - This means it can be read through. Such as "undefined" OR a special object indicator.
+        - Conventional accesses consider these equivalent to having no value, special accesses such as
+            Object.keys() use these to record objects inside of lookups.
+        - An optimization, to save time deserializing the value (however now that we have it, it's required,
+            and MUST be set for undefined values).
+        - Is compatible with other undefined values when locks are involved (for example, if you require
+            the value to be undefined, and you see a isTransparent, that's fine, and your lock will
+            be considered valid).
+    */
+    isTransparent?: boolean;
+
+
+    /** Authorities will clobber this, but non-authorities might use this?
+     *      - This is mostly an in-memory value, and not meant for storage or transmission.
+     *  IMPORTANT! The ONLY reason a PathValue can be rejected is if one of it's locks are rejected
+     *      (due to a conflict, or that lock value being rejected itself). See call prediction for
+     *      how this can be done to allow arbitrary rejection (as in, locking a large time range,
+     *      on a unique path, and then writing to that path when we want to cause a rejection).
+     */
+    valid?: boolean;
+
+    // #region Serialization Only
+
+    /** ONLY for use by pathValueSerializer */
+    isValueLazy?: boolean;
+
+    source?: string;
+
+    /** Used internally by archive management services. */
+    updateCount?: number;
+
+    // #endregion
+};
+
+export type NodeId = string;
+
+export type WatchConfig = {
+    paths: string[];
+    parentPaths: string[];
+};
+
+export type PathValueSnapshot = {
+    values: { [path: string]: PathValue[] };
+};
+
+
+export function getNextTime(): Time {
+    return {
+        time: getNextTimeInternal(),
+        version: 0,
+        creatorId: getCreatorId(),
+    };
+}
+export function compareTime(lhs: Time, rhs: Time) {
+    let diff = lhs.time - rhs.time;
+    if (diff !== 0) return diff;
+    diff = lhs.version - rhs.version;
+    if (diff !== 0) return diff;
+    return lhs.creatorId - rhs.creatorId;
+}
+
+export function getCreatorId() {
+    return IdentityController_getOwnPubKeyShort();
+}
+let prevTime = 0;
+function getNextTimeInternal() {
+    let time = Date.now();
+    if (time <= prevTime) {
+        time = addEpsilons(prevTime, 1);
+    }
+    prevTime = time;
+    return time;
+}
+
+export let debugRejections = false;
+export function enableDebugRejections() {
+    debugRejections = true;
+}
+export function disableDebugRejections() {
+    debugRejections = false;
+}
+export let isCoreQuiet = true;
+export function quietCoreMode() {
+    isCoreQuiet = true;
+}
+export function debugCoreMode() {
+    isCoreQuiet = false;
+}
+
+export function timeHash(time: Time) {
+    return time.creatorId + "_" + time.time;
+}
+export function getLockHash(lock: ReadLock) {
+    // NOTE: startTime might be 0, in which case the path is required in the hash.
+    return timeHash(lock.startTime) + "_" + timeHash(lock.endTime) + "_" + lock.path;
+}
+
+export function debugPathValuePath(pathValue: { time: Time; path: string }): string {
+    // Log the raw path, so it can show up in searches
+    return `(${debugTime(pathValue.time)}) ${pathValue.path}`;
+}
+export function debugPathValue(pathValue: PathValue, write?: boolean): string {
+    return `${debugPathValuePath(pathValue)} ${write ? "=" : "==="} ${String(pathValueSerializer.getPathValue(pathValue))}`;
+}
+export function debugTime(time: Time): string {
+    return `${time.time}[${time.version}]@${time.creatorId.toString().slice(4, 12)}`;
+}
+
+export function filterChildPaths(parentPath: string, packedSuffix: string, paths: Set<string> | undefined) {
+    if (!packedSuffix || !paths) return paths;
+    return filterChildPathsBase(parentPath, packedSuffix, paths);
+}
+
+/** Returns a number between 0 (inclusive) and 1 (exclusive)
+ *      - See matchesParentRangeFilter, matchesParentRangeFilterPart, and filterChildPaths.
+ *  FOR INTERNAL USE ONLY!
+ */
+export function __getRoutingHash(key: string): number {
+    let hash = sha256(key);
+    return getBufferFraction(Buffer.from(hash, "hex"));
+}
+// NOTE: Assumes fullPath is already a child of parentPath, and only checks for hash
+export function matchesParentRangeFilter(config: {
+    parentPath: string;
+    fullPath: string;
+    start: number;
+    end: number;
+}) {
+    if (config.start === 0 && config.end === 1) return true;
+    let part = getPathIndexAssert(config.fullPath, getPathDepth(config.parentPath));
+    let hash = __getRoutingHash(part);
+    return config.start <= hash && hash < config.end;
+}
+export function matchesParentRangeFilterPart(config: {
+    part: string;
+    start: number;
+    end: number;
+}) {
+    if (config.start === 0 && config.end === 1) return true;
+    let hash = __getRoutingHash(config.part);
+    return config.start <= hash && hash < config.end;
+}
+
+let getCompressNetworkBase = () => false;
+export const registerGetCompressNetwork = (fnc: () => boolean) => { getCompressNetworkBase = fnc; };
+export function getCompressNetwork() {
+    return getCompressNetworkBase();
+}
+
+let getCompressDiskBase = () => false;
+export const registerGetCompressDisk = (fnc: () => boolean) => { getCompressDiskBase = fnc; };
+export function getCompressDisk() {
+    return getCompressDiskBase();
+}
+
+let isOurPrediction = (value: PathValue) => false;
+export const registerIsOurPrediction = (fnc: (value: PathValue) => boolean) => { isOurPrediction = fnc; };
+
+const filterChildPathsBase = measureWrap(
+    function filterChildPathsBase(parentPath: string, packedSuffix: string, paths: Set<string>): Set<string> {
+        let [startFractionStr, endFractionStr] = packedSuffix.split("|");
+        let startFraction = Number(startFractionStr);
+        let endFraction = Number(endFractionStr);
+
+        let depth = getPathDepth(parentPath);
+
+        let filtered = new Set<string>();
+        for (let path of paths) {
+            let key = getPathIndexAssert(path, depth);
+            let hash = __getRoutingHash(key);
+            if (startFraction <= hash && hash < endFraction) {
+                filtered.add(path);
+            }
+        }
+        //console.log(`Filtered ${paths.size} paths to ${filtered.size} paths`);
+        return filtered;
+    }
+);
+export function encodeParentFilter(config: {
+    path: string;
+    startFraction: number;
+    endFraction: number;
+}) {
+    return hack_setPackedPathSuffix(config.path, `${config.startFraction}|${config.endFraction}`);
+}
+export function decodeParentFilter(path: string): { start: number, end: number } | undefined {
+    let packedSuffix = hack_getPackedPathSuffix(path);
+    if (!packedSuffix) return undefined;
+    let [startStr, endStr] = packedSuffix.split("|");
+    return { start: Number(startStr), end: Number(endStr) };
+
+}
+
+class AuthorityPathValueStorage {
+    // path => sorted by -time (so newest are first)
+    //  NOTE: We automatically drop historical values that can no longer influence
+    //      the most current state.
+    private values = registerMapArrayResource("paths|authorityPathValueStorage", new Map<string, PathValue[]>());
+    // parent => child paths
+    private parentPathLookup = registerResource("paths|parentPathLookup", new Map<string, Set<string>>());
+    private isSyncedCache = registerResource("paths|isSyncedCache", new Set<string>());
+    private parentsSynced = registerResource("paths|parentsSynced", new Map<string,
+        true |
+        // If set, contains the nodes we have yet to sync before we can consider the parent synced.
+        Set<string>
+    >());
+
+    public DEBUG_UNWATCH = false;
+
+    private overrides = new Set<Map<string, PathValue>>();
+
+    /** The latest times of paths, that have been deleted (but aren't necessarily still deleted) */
+    private lastDeleteAt = new Map<string, Time>();
+    private lastDeleteAtOld = new Map<string, Time>();
+    private lastDeleteAtOlder = new Map<string, Time>();
+    private discardOldLastDeletes = (() => {
+        const self = this;
+        runInfinitePoll(ARCHIVE_FLUSH_LIMIT, function discardOldLastDeletes() {
+            self.lastDeleteAtOlder = self.lastDeleteAtOld;
+            self.lastDeleteAtOld = self.lastDeleteAt;
+            self.lastDeleteAt = new Map();
+        });
+    })();
+
+    // NOTE: All of our functions only return valid values (unless otherwise specified).
+    //  - epochTime reads the latest value, which is important, as we may receive values
+    //      with time > Date.now(), in the case that our clock is behind the server.
+
+    public getValue(path: string): unknown {
+        return pathValueSerializer.getPathValue(this.getValueAtTime(path));
+    }
+    public getValueExactIgnoreInvalid(path: string, time: Time): PathValue | undefined {
+        let value = this.getValueAtTime(path, time, true);
+        if (value && compareTime(value.time, time) === 0) {
+            return value;
+        }
+        return undefined;
+    }
+
+    /** time === undefined gets the latest value */
+    public getValueAtTime(
+        path: string,
+        // Include all values with other.time <= time, returning the one with the highest time
+        //      (either that is valid, or, if readInvalid is true, just the highest time).
+        time?: Time,
+        readInvalid = false
+    ): PathValue | undefined {
+        let overrideValue: PathValue | undefined;
+        if (this.overrides.size > 0) {
+            for (let override of this.overrides) {
+                let value = override.get(path);
+                if (value) {
+                    if (
+                        (!time || compareTime(value.time, time) <= 0)
+                        && (!overrideValue || compareTime(value.time, overrideValue.time) > 0)
+                    ) {
+                        overrideValue = value;
+                    }
+                }
+            }
+        }
+
+        let values = this.values.get(path);
+        if (values) {
+            for (let value of values) {
+                if (!value.valid && !readInvalid) continue;
+                if (!time || compareTime(value.time, time) <= 0) {
+                    if (overrideValue && compareTime(overrideValue.time, value.time) >= 0) {
+                        return overrideValue;
+                    }
+                    return value;
+                }
+            }
+        }
+        if (overrideValue) return overrideValue;
+        return this.getValueOrEpochIfSynced(path);
+    }
+    public getValueBeforeTime(path: string, time: Time): PathValue | undefined {
+        return this.getValueAtTime(path, timeMinusEpsilon(time));
+    }
+    public getValueOrEpochIfSynced(path: string): PathValue | undefined {
+        if (this.isSynced(path)) {
+            return { path, valid: true, time: epochTime, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true };
+        }
+        return undefined;
+    }
+    // Returns at least one value (if any are available)
+    public getValuePlusHistory(path: string, time?: number): PathValue[] {
+        if (this.overrides.size > 0) {
+            throw new Error(`AuthorityPathValueStorage.getValuePlusHistory support for temporaryOverride not implemented yet.`);
+        }
+        let valuesToReturn: PathValue[] = [];
+        let values = this.values.get(path);
+        if (values) {
+            for (let value of values) {
+                if (!value.valid) continue;
+                if (valuesToReturn.length > 0 && (!time || value.time.time < time)) break;
+                valuesToReturn.push(value);
+            }
+        }
+        return valuesToReturn;
+    }
+
+    // TODO: Optimize this via an index
+    /** NOTE: Returns paths that have undefined values. If you want to exclude those
+     *      you will need to check the values yourself (as this function does not accept a time argument).
+     *      TODO: We SHOULD probably check for undefined values here... as it is surely faster than checking
+     *          anywhere else, as we already have the values array!
+     */
+    @measureFnc
+    public getPathsFromParent(parentPath: string): Set<string> | undefined {
+        let packedSuffix = hack_getPackedPathSuffix(parentPath);
+        parentPath = hack_stripPackedPath(parentPath);
+        let result = this.parentPathLookup.get(parentPath);
+        return filterChildPaths(parentPath, packedSuffix, result);
+    }
+
+    @measureFnc
+    private addParentPath(path: string) {
+        let parentPath = getParentPathStr(path);
+        let childPaths = this.parentPathLookup.get(parentPath);
+        if (!childPaths) {
+            childPaths = new Set();
+            this.parentPathLookup.set(parentPath, childPaths);
+        }
+        childPaths.add(path);
+    }
+
+    /** Called by PathWatcher / ourself when paths are unwatched / destroyed */
+    public markPathAsUnwatched(path: string) {
+        // If we are the authority we:
+        //  1) Need to maintain the values for other nodes
+        //  2) Don't need to worry about the values getting out of date,
+        //      because we will get any new values whether we want to or not!
+        if (pathValueAuthority2.isSelfAuthority(path)) return;
+
+        if (this.DEBUG_UNWATCH) {
+            console.log(blue(`Unsyncing path at ${Date.now()}`), path);
+        }
+
+        this.isSyncedCache.delete(path);
+        this.removePathFromStorage(path, "unwatched");
+        writeValidStorage.deleteRemovedPath(path);
+    }
+    public markParentPathAsUnwatched(path: string) {
+        // NOTE: I don't think we have to handle the case where we are the authority,
+        //  because we don't delete any actual data here.
+        this.parentsSynced.delete(path);
+    }
+
+    /** Used for atomic operations, to ensure a path is stable enough (and to check that it hasn't
+     *      changed during the course of an operation).
+     */
+    public getLastChangedTime(path: string): Time | undefined {
+        let time = this.values.get(path)?.[0]?.time;
+        let time0 = this.lastDeleteAt.get(path);
+        let time1 = this.lastDeleteAtOld.get(path);
+        let time2 = this.lastDeleteAtOlder.get(path);
+
+        if (!time || (time0 && compareTime(time0, time) > 0)) time = time0;
+        if (!time || (time1 && compareTime(time1, time) > 0)) time = time1;
+        if (!time || (time2 && compareTime(time2, time) > 0)) time = time2;
+
+        return time;
+    }
+
+
+    @measureFnc
+    public async getSnapshot(authority: AuthorityPath): Promise<PathValueSnapshot> {
+        let snapshot: PathValueSnapshot = {
+            values: Object.create(null),
+        };
+        let now = Date.now();
+        let count = 0;
+        for (let [path, values] of this.values) {
+            count++;
+            if (count > 10_000) {
+                await delay("paintLoop");
+                count = 0;
+            }
+            // NOTE: We can't return extra values, as they might be outdated values which were
+            //  set to undefined, and then deleted, so returning them will create spurious values.
+            if (!pathValueAuthority2.isInAuthority(authority, path)) continue;
+            this.clearRedundantOldValues(values, now);
+            if (values.length === 0) continue;
+            snapshot.values[path] = values;
+        }
+        return snapshot;
+    }
+    @measureFnc
+    public ingestSnapshot(snapshot: PathValueSnapshot) {
+        // NOTE: It is important that we don't try to archive these values, as they either came
+        //  from the archive, or have already been archived.
+
+        let now = Date.now();
+
+        for (let [path, values] of Object.entries(snapshot.values)) {
+            let existingValues = this.values.get(path);
+            for (let value of values) {
+                if (value.canGCValue) {
+                    this.possiblyUndefinedPaths().value.add(value.path);
+                }
+                if (value.event) {
+                    this.eventPaths().value.add(value.path);
+                }
+            }
+            if (!existingValues) {
+                this.values.set(path, values);
+                this.addParentPath(path);
+            } else {
+                let allValues = existingValues.concat(values);
+                allValues.sort((a, b) => compareTime(b.time, a.time));
+                // Remove duplicates
+                for (let i = 0; i < allValues.length - 1; i++) {
+                    if (allValues[i].time.time === allValues[i + 1].time.time &&
+                        allValues[i].time.creatorId === allValues[i + 1].time.creatorId) {
+                        allValues.splice(i, 1);
+                        i--;
+                    }
+                }
+                values = allValues;
+            }
+            this.clearRedundantOldValues(values, now);
+        }
+    }
+
+    public resetForInitialTrigger(path: string) {
+        if (isDebugLogEnabled()) {
+            debugLog("REMOVE FOR INITIAL SYNC", { path });
+        }
+        this.removePathFromStorage(path, "reset for initial trigger");
+    }
+
+    public DEBUG_hasAnyValues(path: string) {
+        let values = this.values.get(path);
+        if (!values) return false;
+        if (values.length === 1 && compareTime(values[0].time, epochTime) === 0) {
+            return false;
+        }
+        return true;
+    }
+    public isSynced(path: string) {
+        if (pathValueAuthority2.isSelfAuthority(path)) return true;
+
+        if (this.isSyncedCache.has(path)) return true;
+
+        // It is only synced if we have values that we DIDN'T create (aka, predictions)
+        let values = this.values.get(path);
+        if (values?.some(x => !isOurPrediction(x))) {
+            if (!isCoreQuiet && isNode()) {
+                //console.log(`Marking path as synced`, path);
+            }
+            this.isSyncedCache.add(path);
+            return true;
+        }
+        return false;
+    }
+    public isParentSynced(path: string) {
+        // We have to check a test child path to see if we are a self authority
+        if (pathValueAuthority2.isSelfAuthority(appendToPathStr(path, ""))) return true;
+
+        return this.parentsSynced.get(path) === true;
+    }
+
+    private getMultiNodesForParent: (path: string) => Map<string, unknown> | undefined = () => undefined;
+    public setGetMultiNodesForParent(fnc: (path: string) => Map<string, unknown> | undefined) {
+        this.getMultiNodesForParent = fnc;
+    }
+    @measureFnc
+    public ingestValues(newValues: PathValue[], parentsSynced: string[] | undefined, parentSyncedSources: Map<string, string[]> | undefined) {
+        let now = Date.now();
+
+        if (isDebugLogEnabled()) {
+            for (let value of newValues) {
+                debugLog("INGEST VALUE", { path: value.path, time: value.time.time });
+            }
+        }
+
+        if (parentsSynced) {
+            for (let parentPath of parentsSynced) {
+                let prevSynced = this.parentsSynced.get(parentPath);
+                if (prevSynced === true) continue;
+                let remoteNodeIds = this.getMultiNodesForParent(parentPath);
+                let sourceNodeIds = parentSyncedSources?.get(parentPath);
+                // NOTE: If there is only 1 authority, assume we received data from it, as who else would we receive it from?
+                if (!remoteNodeIds || remoteNodeIds.size === 1 || !sourceNodeIds) {
+                    this.parentsSynced.set(parentPath, true);
+                } else {
+                    if (!prevSynced) {
+                        prevSynced = new Set(remoteNodeIds.keys());
+                        this.parentsSynced.set(parentPath, prevSynced);
+                    }
+                    for (let sourceNodeId of sourceNodeIds) {
+                        prevSynced.delete(sourceNodeId);
+                    }
+                    if (prevSynced.size === 0) {
+                        this.parentsSynced.set(parentPath, true);
+                    }
+                }
+            }
+        }
+
+        for (let i = 0; i < newValues.length; i++) {
+            let value = newValues[i];
+            if (value.event) {
+                this.eventPaths().value.add(value.path);
+            }
+            if (value.canGCValue) {
+                this.possiblyUndefinedPaths().value.add(value.path);
+            }
+
+            if (!value.event && !value.path.startsWith(LOCAL_DOMAIN_PATH)) {
+                let authorityPath = pathValueAuthority2.getSelfArchiveAuthority(value.path);
+
+                // Store in pending archive lookup
+                if (authorityPath) {
+                    let values = this.pendingArchiveValues.get(authorityPath);
+                    if (!values) {
+                        values = [];
+                        this.pendingArchiveValues.set(authorityPath, values);
+                    }
+                    // If our value has no locks it can never be rejected, so if it is the latest, don't bother to keep the others
+                    if (value.lockCount === 0 && values.length > 0 && values.every(x => compareTime(x.time, value.time) < 0)) {
+                        validateLockCount(value);
+                        if (values.length > 1) {
+                            values.splice(1, values.length - 1);
+                        }
+                        values[0] = value;
+                    } else {
+                        values.push(value);
+                    }
+                }
+            }
+
+            // Store in global lookup
+            {
+                let specialGoldenCase = false;
+                let values = this.values.get(value.path);
+                if (!values) {
+                    values = [];
+                    this.values.set(value.path, values);
+                    this.addParentPath(value.path);
+                }
+                let insertIndex = values.findIndex(x => compareTime(x.time, value.time) <= 0);
+                if (insertIndex < 0) {
+                    values.push(value);
+                } else {
+                    let prev = values[insertIndex];
+                    if (prev && compareTime(prev.time, value.time) === 0) {
+                        // Just update the valid state (we likely aren't an authority on the path anyways)
+                        prev.valid = value.valid;
+                    } else {
+                        // Special case values that always have no locks, by just setting the value, and never splicing
+                        if (insertIndex === 0 && value.lockCount === 0 && values.length <= 1) {
+                            validateLockCount(value);
+                            if (!isCoreQuiet) {
+                                if (values.length === 1) {
+                                    console.log(`Clobbering ${debugPathValuePath(values[0])} with ${debugPathValuePath(value)}`);
+                                }
+                            }
+                            specialGoldenCase = true;
+                            values[0] = value;
+                        } else {
+                            values.splice(insertIndex, 0, value);
+                        }
+                    }
+                }
+                if (!specialGoldenCase) {
+                    this.clearRedundantOldValues(values, now);
+                }
+            }
+        }
+
+        // NOTE: Archiving takes a long time (we buffer values for at least a few minutes), so we can't
+        //  wait for them to archive. We don't retry on storage failure, because... we should have enough
+        //  nodes that received it to have the values committed in the long term. Unless we are the only
+        //  node, in which case redundancy clearly isn't that important anyway (also, if we are only running
+        //  one node we are likely using the local disk, so writes really shouldn't fail! And if they do,
+        //  it is likely because the disk is full, and so retrying wouldn't help anyways!)
+
+        this.archiveLoop();
+    }
+
+    // authorityPath => values
+    private pendingArchiveValues = registerMapArrayResource("pendingArchiveValues", new Map<AuthorityPath, PathValue[]>());
+    private shuttingDown = new PromiseObj<"shutdown">();
+    private pendingShutdownWrites: PromiseObj | undefined;
+
+    private archiveLoop = lazy(() => {
+        // No need to even run the archive loop in the browser...
+        if (!isNode()) return;
+        if (yargObj.noarchive) return;
+        let self = this;
+        runInfinitePoll(ARCHIVE_LOOP_DELAY, async function archiveLoop() {
+            if (self.shuttingDown.resolved) {
+                return;
+            }
+            await self.innerArchiveLoop();
+        });
+    });
+    private async innerArchiveLoop() {
+        let pending = this.pendingArchiveValues;
+        this.pendingArchiveValues = new Map();
+        if (pending.size === 0) return;
+
+        // Wait long enough for all values to be old enough to have a fixed valid state.
+        // Also wait some extra time randomly, to reduce the chances of archive being
+        //  called at the same time on multiple nodes at once.
+        let delayTime = MAX_CHANGE_AGE * 2 + Math.random() * MAX_CHANGE_AGE * 0.5;
+        if (this.shuttingDown.resolved) {
+            delayTime = 0;
+        }
+
+        if (!isCoreQuiet || this.shuttingDown.resolved) {
+            console.log(blue(`Archiving ${pending.size} paths in ${formatTime(delayTime)}`));
+        }
+
+        let inShutdown = await PromiseRace([this.shuttingDown.promise, delay(delayTime)]);
+        if (inShutdown === "shutdown") {
+            this.pendingShutdownWrites = new PromiseObj();
+        }
+
+        try {
+            for (let [authorityPath, allValues] of pending) {
+                let validValues = allValues
+                    .filter(x => x.valid)
+                    // ALSO, don't archive epochTime values...
+                    .filter(x => x.time.time)
+                    ;
+                await pathValueArchives.archiveValues(authorityPath, validValues);
+            }
+        } finally {
+            this.pendingShutdownWrites?.resolve();
+        }
+
+    }
+    public async onShutdown() {
+        if (yargObj.noarchive) return;
+        this.shuttingDown.resolve("shutdown");
+        await pathValueAuthority2.waitUntilRoutingIsReady();
+        // If there are other nodes, we CAN'T just archive our values, as some may be rejected in the future.
+        let nodes = pathValueAuthority2.getReadNodes(rootPathStr);
+        let otherReadNodes = nodes.filter(x => !isOwnNodeId(x));
+        if (otherReadNodes.length > 0 && this.pendingArchiveValues.size > 0) {
+            console.log(`Other authorities (${otherReadNodes.length}), discarding ${this.pendingArchiveValues.size} pending writes`);
+        }
+        if (otherReadNodes.length === 0) {
+            // Wait the PromiseRace to finish
+            await delay("afterpromises");
+            // Wait for current loop to finish
+            await this.pendingShutdownWrites?.promise;
+            // Run another loop, in case a loop wasn't running
+            await this.innerArchiveLoop();
+        }
+    }
+
+
+    // NOTE: I'm not sure we really need to call this. It gets caller per write, which should be sufficient.
+    //  The only case we leak is if there are a lot of writes on a value, then none. But that shouldn't
+    //  really happen too often (and the real issue is not moving values from memory => storage, which when we do,
+    //  we will call this anyways).
+    // NOTE: values are sorted sorted by -time (so newest are first)
+    public clearRedundantOldValues(values: PathValue[], now: number) {
+        // NOTE: We can also delete golden rejections BEFORE the valid golden value. But...
+        //  that is less important, as there are likely to be few rejected values, so this
+        //  is really just designed for removing older golden values (which are probably valid).
+
+        // NOTE: We clear values as soon as we can to make FunctionRunners run faster, as any database
+        //  with many writes will almost certainly have many FunctionRunner calls, which all need to synchronize
+        //  as much history as we have. So... the less history, we faster FunctionRunners can sync. AND syncing
+        //  data is the true bottleneck, as it requires network traffic, vs this, which is just manipulating
+        //  some values in memory...
+
+        let goldenTime = now - MAX_CHANGE_AGE;
+        // NOTE: Values with no locks are implicitly golden, as they can't be rejected.
+        let firstGoldenIndex = values.findIndex(x => x.time.time < goldenTime || x.lockCount === 0);
+        if (firstGoldenIndex < 0) return;
+        // Special case, everything is golden, and the latest value is undefined, and is behind the GC
+        //  point, then delete everything, so constantly changing keys doesn't result in us leaking memory forever.
+        if (
+            firstGoldenIndex === 0
+            && values[0].canGCValue
+            && values[0].time.time < now - VALUE_GC_THRESHOLD
+        ) {
+            if (!isCoreQuiet) {
+                console.log(`GCing ALL ${debugPathValuePath(values[0])}`);
+            }
+            // Mark the value as synced though, as synced means we have received values for this path,
+            //  and while the values are undefined and so we are deleting them... we have definitely
+            //  received values!
+            let path = values[0].path;
+            this.isSyncedCache.add(path);
+            this.removePathFromStorage(path, "value is undefined and golden");
+            values.splice(0, values.length);
+            return;
+        }
+        let goldenCount = values.length - firstGoldenIndex;
+        // We can only compress golden values, so if there is only 1 golden, then we can't compress it
+        //  (as we can't remove it, as the live value might become rejected)
+        if (goldenCount < 2) return;
+
+        if (firstGoldenIndex === 0) {
+            // Remove everything but the latest value
+            let removed = values.splice(1, values.length - 1);
+            if (!isCoreQuiet) {
+                console.log(`GCing everything older than ${debugPathValuePath(values[0])}`);
+                for (let value of removed) {
+                    console.log(`    ${debugPathValuePath(value)}`);
+                }
+            }
+        } else {
+            // Delete everything that is golden, except for the most recent VALID value.
+            let validGoldenIndex = values.findIndex(x => x.valid && x.time.time < goldenTime);
+            if (validGoldenIndex >= 0 && values.length > validGoldenIndex) {
+                for (let i = values.length - 1; i > validGoldenIndex; i--) {
+                    let gced = values.pop();
+                    if (!isCoreQuiet && gced) {
+                        console.log(`GCing ${debugPathValuePath(gced)}`);
+                    }
+                }
+            }
+        }
+    }
+
+    private removePathFromStorage(path: string, reason: string) {
+        let values = this.values.get(path);
+        if (values?.length) {
+            this.lastDeleteAt.set(path, values[0].time);
+        }
+        this.values.delete(path);
+
+        if (isDebugLogEnabled()) {
+            debugLog("REMOVE PATH", { path, reason });
+        }
+
+        // NOTE: parentPathLookup is important because clients often use Object.keys() to get paths,
+        //  and then watch all of them (which incurs constant permissions checks, etc). While isSyncedCached
+        //  and parentsSynced are just in memory, are never iterated, so extra values there are far less important...
+        let parentPath = getParentPathStr(path);
+        let childPaths = this.parentPathLookup.get(parentPath);
+        if (childPaths) {
+            childPaths.delete(path);
+            if (childPaths.size === 0) {
+                this.parentPathLookup.delete(parentPath);
+            }
+        }
+    }
+
+    private possiblyUndefinedPaths = lazy((): { value: Set<string> } => {
+        let pathsToCheckPointer = { value: new Set<string>() };
+
+        runInfinitePoll(UNDEFINED_MEMORY_CLEANUP_DELAY, () => {
+            let now = Date.now();
+            let deadTime = now - UNDEFINED_MEMORY_CLEANUP_DELAY;
+            let pathsToCheck = pathsToCheckPointer.value;
+            let nextPathsToCheck = new Set<string>();
+            pathsToCheckPointer.value = nextPathsToCheck;
+            for (let path of pathsToCheck) {
+                let values = this.values.get(path);
+                if (!values?.length) continue;
+                let currentValue = values[0];
+                if (!currentValue.canGCValue) continue;
+
+                // NOTE: We can delete invalid values just the same as valid values, so we don't
+                //  even check the valid state
+
+                if (currentValue.time.time >= deadTime) {
+                    // Give it some time, it will probably stay deleted, but will be old enough
+                    //  to delete the next time we check (or have a value, and then we will stop checking it).
+                    nextPathsToCheck.add(path);
+                    continue;
+                }
+
+                // NOTE: We have to set isSynced otherwise our render functions won't think they can render
+                //  (also proxies will fail to forward epoch values at all).
+                if (pathWatcher.isWatching(path)) {
+                    this.isSyncedCache.add(path);
+                }
+
+                // It is equivalent to delete the value than to keep it, so... just delete it to save space
+                //  - We don't need to worry about new locks on the value, as any new locks will be assumed
+                //      valid, due to the fact that if a value is old enough, it is either valid,
+                //      or it is old enough that the server will know it is invalid, and not send it.
+                this.removePathFromStorage(path, "undefined and golden from loop");
+            }
+        });
+        return pathsToCheckPointer;
+    });
+
+    private eventPaths = lazy((): { value: Set<string> } => {
+        let pathsToCheckPointer = { value: new Set<string>() };
+
+        let garbageCollectTime = MAX_CHANGE_AGE * 2;
+        runInfinitePoll(garbageCollectTime, () => {
+            let deadTime = Date.now() - garbageCollectTime;
+            let pathsToCheck = pathsToCheckPointer.value;
+            let nextPathsToCheck = new Set<string>();
+            pathsToCheckPointer.value = nextPathsToCheck;
+            let pathsToClear = new Set<string>();
+            for (let path of pathsToCheck) {
+                let values = this.values.get(path);
+                if (!values) {
+                    pathsToClear.add(path);
+                    continue;
+                }
+                let value = values[values.length - 1];
+                if (!value) {
+                    pathsToClear.add(path);
+                    continue;
+                }
+                if (value.time.time < deadTime) {
+                    pathsToClear.add(path);
+                } else {
+                    // Give it more time
+                    nextPathsToCheck.add(path);
+                }
+            }
+
+            if (pathsToClear.size > 0) {
+                for (let path of pathsToClear) {
+                    this.markPathAsUnwatched(path);
+                    // I'm not sure if removing it as a parent is needed, or... maybe it is needed,
+                    //  and this isn't enough of a check?
+                    this.markParentPathAsUnwatched(path);
+                    // And... remove from storage, in case we are the authority and wouldn't normally remove it.
+                    this.removePathFromStorage(path, "event path");
+                }
+                pathWatcher.unwatchEventPaths(pathsToClear);
+                for (let callback of this.eventRemovalsCallbacks) {
+                    callback(pathsToClear);
+                }
+            }
+        });
+        return pathsToCheckPointer;
+    });
+
+    private eventRemovalsCallbacks = new Set<(removedPaths: Set<string>) => void>();
+    public watchEventRemovals(callback: (removedPaths: Set<string>) => void) {
+        this.eventRemovalsCallbacks.add(callback);
+    }
+
+    /** All overrides are assumed to be valid. */
+    public temporaryOverride<T>(values: PathValue[] | undefined, code: () => T): T {
+        if (!values) return code();
+        let overrideMap = new Map(values.map(x => [x.path, x]));
+        this.overrides.add(overrideMap);
+        try {
+            return code();
+        } finally {
+            this.overrides.delete(overrideMap);
+        }
+    }
+
+    /** For value auditing. Might change to expose less, so don't use this for anything except auditting. */
+    public __auditValues() {
+        return this.values;
+    }
+}
+export const authorityStorage = new AuthorityPathValueStorage();
+
+
+// If PathValue, it is a local watch
+export type WriteCallback = NodeId | PathValue[];
+
+// isOwnNodeId(nodeId) means localOnValueCallback is called (instead of PathValueController.forwardWrites)
+export type PathWatcherCallback = NodeId;
+class PathWatcher {
+    private watchers = registerResource("paths|PathWatcher.watchers", new Map<string, Set<PathWatcherCallback>>());
+    // realPath => packedPath => watcher => { depth, start, end }[]
+    private parentWatchers = registerResource("paths|parentWatchers", new Map<string,
+        // packedPath
+        Map<string, {
+            watchers: Set<PathWatcherCallback>;
+            start: number;
+            end: number;
+        }>
+    >());
+    private watchersToPaths = registerResource("paths|watchersToPaths", new Map<PathWatcherCallback, {
+        paths: Set<string>;
+        parents: Set<string>;
+    }>());
+
+    /** Automatically unwatches on callback disconnect */
+    @measureFnc
+    public watchPath(config: WatchConfig & {
+        callback: PathWatcherCallback;
+        debugName?: string;
+        noInitialTrigger?: boolean;
+        /** Force trigger, even if we are already watching the values. */
+        initialTrigger?: boolean;
+    }) {
+        if (config.callback) {
+            this.ensureUnwatchingOnDisconnect(config.callback);
+        }
+
+        let newPathsWatched = new Set<string>();
+        let newParentsWatched = new Set<string>();
+
+        for (let path of config.paths) {
+            let watchers = this.watchers.get(path);
+            if (!watchers) {
+                watchers = new Set();
+                this.watchers.set(path, watchers);
+            }
+            if (watchers.has(config.callback)) continue;
+            newPathsWatched.add(path);
+            watchers.add(config.callback);
+
+            let watchObj = this.watchersToPaths.get(config.callback);
+            if (!watchObj) {
+                watchObj = { paths: new Set(), parents: new Set() };
+                this.watchersToPaths.set(config.callback, watchObj);
+            }
+            watchObj.paths.add(path);
+        }
+        for (let path of config.parentPaths) {
+            let basePath = hack_stripPackedPath(path);
+            let watchersObj = this.parentWatchers.get(basePath);
+            if (!watchersObj) {
+                watchersObj = new Map();
+                this.parentWatchers.set(basePath, watchersObj);
+            }
+            let obj = watchersObj.get(path);
+            if (!obj) {
+                let range = decodeParentFilter(path) || { start: 0, end: 1 };
+                obj = { watchers: new Set(), start: range.start, end: range.end };
+                watchersObj.set(path, obj);
+            }
+            if (obj.watchers.has(config.callback)) continue;
+            obj.watchers.add(config.callback);
+
+            newParentsWatched.add(path);
+
+            let watchObj = this.watchersToPaths.get(config.callback);
+            if (!watchObj) {
+                watchObj = { paths: new Set(), parents: new Set() };
+                this.watchersToPaths.set(config.callback, watchObj);
+            }
+            watchObj.parents.add(path);
+        }
+
+        if (newPathsWatched.size > 0 || newParentsWatched.size > 0) {
+            diskLog(`New PathValue watches`, {
+                newPathsWatched: newPathsWatched.size,
+                newParentsWatched: newParentsWatched.size,
+            });
+        }
+
+        // Treat everything as a new watch
+        if (config.initialTrigger) {
+            newPathsWatched = new Set(config.paths);
+            newParentsWatched = new Set(config.parentPaths);
+        }
+
+        if (!config.noInitialTrigger) {
+            // Trigger all initial values (for paths we have synced)
+            let initialValues = new Set<PathValue>();
+            let newPaths = new Set(Array.from(newPathsWatched).filter(x => authorityStorage.isSynced(x)));
+            for (let path of newParentsWatched) {
+                // If we haven't synced the parent, we can't forward it. This is important for proxies
+                //  (when they receive the parent synced they will forward the values).
+                if (!authorityStorage.isParentSynced(path)) continue;
+                let paths = authorityStorage.getPathsFromParent(path);
+                if (paths) {
+                    for (let path of paths) {
+                        newPaths.add(path);
+                    }
+                }
+            }
+            if (newPaths.size > 0 || newParentsWatched.size > 0) {
+                for (let path of newPaths) {
+                    let history = authorityStorage.getValuePlusHistory(path);
+                    for (let value of history) {
+                        initialValues.add(value);
+                    }
+                    if (history.length === 0) {
+                        // We always have to send something, or the client won't know when they are ready to show the data!
+                        initialValues.add({ path, value: undefined, canGCValue: true, valid: true, time: epochTime, locks: [], lockCount: 0, event: false });
+                    }
+                }
+                this.triggerLatestWatcher(
+                    config.callback,
+                    Array.from(initialValues),
+                    config.parentPaths,
+                    config.initialTrigger ? "initialTrigger" : undefined,
+                );
+            }
+        }
+    }
+    // NOTE: Automatically unwatches from remoteWatcher on all paths that are fully unwatched
+    @measureFnc
+    public unwatchPath(config: WatchConfig & { callback: PathWatcherCallback; }) {
+        const { callback } = config;
+
+        let fullyUnwatched: WatchConfig = {
+            paths: [],
+            parentPaths: [],
+        };
+
+        let pathsWatched = this.watchersToPaths.get(callback);
+
+        for (let path of config.paths) {
+            if (pathsWatched) {
+                pathsWatched.paths.delete(path);
+            }
+
+            let watchers = this.watchers.get(path);
+            if (!watchers) continue;
+            watchers.delete(callback);
+            if (watchers.size === 0) {
+                this.watchers.delete(path);
+                fullyUnwatched.paths.push(path);
+                authorityStorage.markPathAsUnwatched(path);
+            }
+        }
+        for (let path of config.parentPaths) {
+            if (pathsWatched) {
+                pathsWatched.parents.delete(path);
+            }
+
+            let unpackedPath = hack_stripPackedPath(path);
+
+            let watchersObj = this.parentWatchers.get(unpackedPath);
+            if (!watchersObj) continue;
+            let obj = watchersObj.get(path);
+            if (!obj) continue;
+            obj.watchers.delete(callback);
+
+            if (obj.watchers.size === 0) {
+                watchersObj.delete(path);
+                this.parentWatchers.delete(path);
+                fullyUnwatched.parentPaths.push(path);
+                authorityStorage.markParentPathAsUnwatched(path);
+
+                if (watchersObj.size === 0) {
+                    this.parentWatchers.delete(unpackedPath);
+                }
+            }
+        }
+
+        if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
+            diskLog(`Unwatched PathValue watches`, {
+                unwatchedPaths: fullyUnwatched.paths.length,
+                unwatchedParents: fullyUnwatched.parentPaths.length,
+            });
+            for (let unwatchCallback of this.unwatchedCallbacks) {
+                unwatchCallback(fullyUnwatched);
+            }
+        }
+    }
+
+    public unwatchEventPaths(paths: Set<string>) {
+        for (let path of paths) {
+            let watchers = this.watchers.get(path);
+            if (watchers) {
+                this.watchers.delete(path);
+                for (let watcher of watchers) {
+                    let watchObj = this.watchersToPaths.get(watcher);
+                    if (watchObj) {
+                        watchObj.paths.delete(path);
+                    }
+                }
+            }
+            let parentWatchers = this.parentWatchers.get(path);
+            if (parentWatchers) {
+                this.parentWatchers.delete(path);
+                for (let { watchers } of parentWatchers.values()) {
+                    for (let watcher of watchers) {
+                        let watchObj = this.watchersToPaths.get(watcher);
+                        if (watchObj) {
+                            watchObj.parents.delete(path);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public triggerValuesChanged(valuesChanged: Set<PathValue>, parentsSynced?: string[], initialTriggers?: { values: Set<PathValue>; parentPaths: Set<string> }) {
+        let changedPerCallbacks = this.getWatchers(valuesChanged, parentsSynced);
+        for (let [watch, changes] of changedPerCallbacks) {
+            if (initialTriggers) {
+                let valuesFromInitialTrigger = new Set<PathValue>();
+                let parentsFromInitialTrigger = new Set<string>();
+                let valuesOther = new Set<PathValue>();
+                let parentsOther = new Set<string>();
+                for (let value of changes) {
+                    if (initialTriggers.values.has(value)) {
+                        valuesFromInitialTrigger.add(value);
+                    } else {
+                        valuesOther.add(value);
+                    }
+                }
+                for (let parent of initialTriggers.parentPaths) {
+                    parentsFromInitialTrigger.add(parent);
+                }
+                if (parentsSynced) {
+                    for (let parent of parentsSynced) {
+                        if (!initialTriggers.parentPaths.has(parent)) {
+                            parentsOther.add(parent);
+                        }
+                    }
+                }
+
+                if (valuesFromInitialTrigger.size > 0 || parentsFromInitialTrigger.size > 0) {
+                    this.triggerLatestWatcher(watch, Array.from(valuesFromInitialTrigger), Array.from(parentsFromInitialTrigger), "initialTrigger");
+                }
+                if (valuesOther.size > 0 || parentsOther.size > 0) {
+                    this.triggerLatestWatcher(watch, Array.from(valuesOther), Array.from(parentsOther));
+                }
+            } else {
+                this.triggerLatestWatcher(watch, Array.from(changes), parentsSynced);
+            }
+        }
+    }
+    public getWatchers<T extends { path: string }>(valuesChanged: Set<T>, parentsSynced?: string[]) {
+        let changedPerCallbacks: Map<PathWatcherCallback, Set<T>> = new Map();
+        for (let value of valuesChanged) {
+            let path = value.path;
+            let latestWatches = this.watchers.get(path);
+
+            function triggerNodeChanged(watcher: NodeId) {
+                let changes = changedPerCallbacks.get(watcher);
+                if (!changes) {
+                    changes = new Set();
+                    changedPerCallbacks.set(watcher, changes);
+                }
+                changes.add(value);
+            }
+
+            if (latestWatches) {
+                for (let watch of latestWatches) {
+                    triggerNodeChanged(watch);
+                }
+            }
+
+            let parentPath = getParentPathStr(path);
+            let latestParentWatches = this.parentWatchers.get(parentPath);
+            if (latestParentWatches) {
+                let pathRoutingHash: number | undefined = undefined;
+                for (let { start, end, watchers } of latestParentWatches.values()) {
+                    if (!matchesParentRangeFilter({
+                        parentPath,
+                        fullPath: path,
+                        start,
+                        end,
+                    })) {
+                        continue;
+                    }
+                    for (let watch of watchers) {
+                        triggerNodeChanged(watch);
+                    }
+                }
+            }
+        }
+        for (let parentPath of parentsSynced ?? []) {
+            let latestParentWatches = this.parentWatchers.get(parentPath);
+            if (latestParentWatches) {
+                for (let { watchers } of latestParentWatches.values()) {
+                    for (let watcher of watchers) {
+                        let changes = changedPerCallbacks.get(watcher);
+                        if (!changes) {
+                            changes = new Set();
+                            changedPerCallbacks.set(watcher, changes);
+                        }
+                    }
+                }
+            }
+        }
+        return changedPerCallbacks;
+    }
+
+    public isWatching(path: string): boolean {
+        return this.watchers.has(path) || this.parentWatchers.has(getParentPathStr(path));
+    }
+
+    @measureFnc
+    private triggerLatestWatcher(
+        watcher: PathWatcherCallback,
+        changes: PathValue[],
+        parentPaths?: string[],
+        initialTrigger?: "initialTrigger"
+    ) {
+        if (isOwnNodeId(watcher)) {
+            for (let callback of this.localTriggerCallbacks) {
+                callback(changes, parentPaths ?? []);
+            }
+        } else {
+            if (!isCoreQuiet) {
+                console.log(`(${Date.now()}) Sending values to client: ${changes.length} (${watcher})`);
+            }
+            this.ensureUnwatchingOnDisconnect(watcher);
+            ignoreErrors((async () => {
+                let allowSource = await isNodeTrusted(watcher) || getNodeIdIP(watcher) === "127.0.0.1";
+                let buffers = await pathValueSerializer.serialize(changes, {
+                    noLocks: true,
+                    compress: getCompressNetwork(),
+                    stripSource: !allowSource,
+                });
+
+                if (isDebugLogEnabled()) {
+                    for (let pathValue of changes) {
+
+                        debugLog("SEND VALUE", { path: pathValue.path, time: pathValue.time.time, nodeId: debugNodeId(watcher), transparent: pathValue.isTransparent, canGC: pathValue.canGCValue });
+                    }
+                }
+                await PathValueController.nodes[watcher].forwardWrites(
+                    buffers,
+                    parentPaths,
+                    undefined,
+                    initialTrigger,
+                );
+            })());
+        }
+    }
+
+    private ensureUnwatchingOnDisconnect = cache((nodeId: string) => {
+        if (isOwnNodeId(nodeId)) return;
+        SocketFunction.onNextDisconnect(nodeId, async () => {
+            this.ensureUnwatchingOnDisconnect.clear(nodeId);
+
+            // Wait while, so Querysub doesn't thrash when clients connect and disconnect
+            //  TODO: We should be smarter about our wait time, waiting less if we have a lot of resource
+            //      pressure, and more (potentially forever), if we don't have much resource pressure
+            //      (memory, cpu, and network).
+            setTimeout(() => {
+                let watches = this.watchersToPaths.get(nodeId);
+                this.watchersToPaths.delete(nodeId);
+                if (watches) {
+                    this.unwatchPath({ paths: Array.from(watches.paths), parentPaths: Array.from(watches.parents), callback: nodeId });
+                }
+            }, MAX_CHANGE_AGE);
+        });
+    });
+
+    private localTriggerCallbacks = new Set<(changes: PathValue[], parentPaths: string[]) => void>();
+    public watchAllLocalTriggers(callback: (changes: PathValue[], parentPaths: string[]) => void) {
+        this.localTriggerCallbacks.add(callback);
+    }
+
+    public unwatchedCallbacks = new Set<(config: WatchConfig) => void>();
+    public watchUnwatched(callback: (config: WatchConfig) => void) {
+        this.unwatchedCallbacks.add(callback);
+    }
+
+    public getAllParentWatches() {
+        return Array.from(this.parentWatchers.keys());
+    }
+}
+export const pathWatcher = new PathWatcher();
+
+
+class WriteValidStorage {
+    // We could have a more efficient storage for non-range, but... we need range storage anyways,
+    //  so... this is fine.
+    // (Not sorted)
+    //  - May have duplicates for the same path (for lockless writes), always with the same valid state.
+    // path => WriteState[]
+    //  WriteState is UNSORTED (it is only kept around for MAX_CHANGE_AGE, so this should be the fastest way to do it)
+    private validStorage = registerMapArrayResource("validStorage", new Map<string, WriteState[]>());
+
+    private delayedInvalidate = ((lock: ReadLock, targetTime: number, valueGroupForPredictionHack: PathValue[]) => {
+        const tryNow = () => {
+            if (Date.now() >= targetTime) {
+                setTimeout(tryNow, Date.now() - targetTime + 50);
+                return;
+            }
+            pathValueCommitter.ingestValidStates(valueGroupForPredictionHack.map(x => ({
+                path: x.path,
+                isValid: true,
+                time: x.time,
+                isTransparent: x.isTransparent || false,
+            })), undefined, "recomputeValidState");
+        };
+        tryNow();
+    });
+
+    public getWriteState(readLock: ReadLock, now: number): WriteState {
+        let isValid = this.isLockValid(readLock, now);
+        return { path: readLock.path, isValid, time: readLock.startTime, isTransparent: readLock.readIsTranparent };
+    }
+    public isLockValid(readLock: ReadLock, now: number, valueGroupForPredictionHack?: PathValue[]): boolean {
+        let isValid = this.isLockValidBase(readLock, now);
+        if (!isValid && readLock.keepRejectedUntil && now < readLock.keepRejectedUntil && valueGroupForPredictionHack) {
+            isValid = true;
+            // TODO: Test this code. We haven't run it, as it is a bit annoying to set up (we need to predict
+            //  writes to a new location). It will be pretty obvious if it is failing, but... I don't think it will?
+            console.log(yellow(`Delaying rejection of ${readLock.path} to try to fix out of order prediction invalidation`));
+            // Trigger a recheck when the keepRejectedUntil time is reached
+            this.delayedInvalidate(readLock, readLock.keepRejectedUntil, valueGroupForPredictionHack);
+        }
+        return isValid;
+    }
+    public isLockValidBase(readLock: ReadLock, now: number): boolean {
+        // If it old enough it must be valid, otherwise how would a client NOT know it was invalid
+        //  (assuming clients clear their reads aftering being disconnecting for long enough).
+        if (readLock.startTime.time < now - MAX_CHANGE_AGE) {
+            return true;
+        }
+
+        let values = this.validStorage.get(readLock.path);
+        let time = readLock.startTime;
+        if (readLock.readIsTranparent) {
+            // EDIT: Actually... isn't this done so that rejections can cascade from other nodes, with our node
+            //  not even evaluating the actual rejection reason, but just directly using the reject value?
+            // // EDIT: I *think* that the range check should be sufficient for undefined read locks.
+            // //  If that is the case, remove isUndefinedAtLock entirely.
+            // //  - The reason the range check is good enough is that is is checking for values. Actually,
+            // //      if we always checked for values the range check would always be sufficient. HOWEVER,
+            // //      the range + valid check is an optimization, so locks don't need to store their values.
+            // //      But in this case, with the weird stuff we are doing with predictions... and the fact that
+            // //      we already store the value... we simply don't need the check.
+            // return true;
+            return this.isUndefinedAtLock(readLock);
+        }
+        if (!values) {
+            // We have no record of any values on this path, so it must have never been committed!
+            return false;
+        }
+        let value = values.find(x => compareTime(x.time, time) === 0);
+        if (!value) {
+            // If it is golden, it was probably just removed, so assume it was accepted
+            //  (nodes only ask for valid states of values they created, or that were
+            //  just created, so this is fine)
+            if (time.time < now - MAX_CHANGE_AGE) {
+                return true;
+            }
+            return false;
+        }
+        return value.isValid;
+    }
+    private isUndefinedAtLock(lock: ReadLock): boolean {
+        let values = this.validStorage.get(lock.path);
+        let time = lock.startTime;
+        if (values) {
+            // Find the write at read time
+            let latestValue: WriteState | undefined;
+            for (let value of values) {
+                // Skip invalid
+                if (!value.isValid) continue;
+                // Skip writes AFTER our read (but not ON our read, as then we skip our read itself!)
+                if (compareTime(value.time, time) > 0) continue;
+                // Take the newest write that would affect us
+                if (!latestValue || compareTime(value.time, latestValue.time) > 0) {
+                    latestValue = value;
+                }
+            }
+            if (latestValue && !latestValue.isTransparent) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private getWritesInRange(lock: ReadLock): WriteState[] | undefined {
+        let values = this.validStorage.get(lock.path);
+        if (!values) return undefined;
+        let times: WriteState[] | undefined;
+        for (let value of values) {
+            if (!value.isValid) continue;
+            // NOTE: This is the same comparison as in getValidStateChangedTriggers
+            if (compareTime(lock.startTime, value.time) < 0 && compareTime(value.time, lock.endTime) < 0) {
+                if (!times) times = [];
+                times.push(value);
+            }
+        }
+        return times;
+    }
+
+    // Returns true if the valid state changed
+    public setWriteValidStateValue(pathValue: PathValue): boolean {
+        this.ensureGarbageCollectOldState();
+
+        let time = pathValue.time;
+        // Clone time, so we aren't referencing any of the original object. This is very important for garbage collection.
+        time = { time: time.time, version: time.version, creatorId: time.creatorId, };
+        return this.setWriteValidState({
+            path: pathValue.path,
+            isValid: pathValue.valid || false,
+            time: time,
+            isTransparent: pathValue.isTransparent || false,
+        });
+    }
+    public setWriteValidState(write: WriteState, lockless?: boolean): boolean {
+        this.ensureGarbageCollectOldState();
+
+        // If lockless then it starts valid, and always stays valid.
+        if (!lockless) {
+            let authorityValue = authorityStorage.getValueExactIgnoreInvalid(write.path, write.time);
+            if (authorityValue) {
+                authorityValue.valid = write.isValid;
+                if (!isCoreQuiet) {
+                    console.log(`Setting valid state of ${debugPathValuePath(authorityValue)} to ${write.isValid}`);
+                }
+            } else {
+                if (isNode()) {
+                    console.error(`Setting valid state of ${write.path}@${debugTime(write.time)} to ${write.isValid}, but the ValuePath was not found. If the ValuePath is found later, it might not have the valid state set correctly.`);
+                }
+            }
+        }
+
+        let path = write.path;
+        let time = write.time;
+        let isValid = write.isValid;
+        let values = this.validStorage.get(path);
+        if (!values) {
+            values = [];
+            this.validStorage.set(path, values);
+        }
+        // NOTE: We always have to search, even if lockless, so we can return false when the isValid state hasn't changed.
+        // Search from the end, as it is most likely time will have been recently added
+        let index = values.length - 1;
+        while (index >= 0) {
+            let diff = compareTime(values[index].time, time);
+            if (diff === 0) {
+                break;
+            }
+            index--;
+        }
+        if (index >= 0) {
+            if (values[index].isValid === isValid) {
+                return false;
+            }
+            values[index].isValid = isValid;
+        } else {
+            values.push(write);
+        }
+        return true;
+    }
+
+
+    /** SOMEWHAT of a hack. Updates the inputs value states from cache. Needed, as it is possible for valid states
+            to be processed before PathValues. Which would cause the PathValues to have outdated valid states when received.
+            Valid states are always correct, and always updated (if they are sent at all), so we will use the latest valid
+            state instead of latest write state.
+    */
+    @measureFnc
+    public updateValidStatesFromCache(values: PathValue[]) {
+        for (let value of values) {
+            let states = this.validStorage.get(value.path);
+            if (!states) continue;
+            let state = states.find(x => compareTime(x.time, value.time) === 0);
+            if (!state) continue;
+            value.valid = state.isValid;
+        }
+    }
+
+    // NOTE: Mutates the input PathValue.valid states to be the new states
+    // NOTE: Only mutates paths we are an authority on
+    // NOTE: Also sets up watches, so the valid states will update when their readLocks change
+    //  (including talking to remote watchers, etc, etc)
+    @measureFnc
+    public computeValidStates(values: PathValue[], now: number, alreadyWatching?: "alreadyWatching"): WriteState[] {
+        // Calculate valid states
+        //  - For things we are not an authority on, assumes their state is true.
+        //      - If we AREN'T an authority, whomever sent us the values will send us new
+        //          values if these become invalid.
+        //          - Also, if we created the values, we will get sent valid states if
+        //              they become invalid.
+        //          - We don't need to watch locks for pathValues, as any valid states changes
+        //              automatically impact any values that use them (we only need to watch locks
+        //              for values we are computing).
+        //  - For everything we ARE an authority on, computes them
+        //      - Also sets up watches for computed values, so the valid states will stay up to date).
+
+        let changes: WriteState[] = [];
+
+        // If alreadyWatching... then these are triggers caused by a local lock watch. In which case,
+        //  are must want to know if the values changed (as in, for call prediction).
+        if (!alreadyWatching) {
+            measureBlock(function computeValidStates_getSelfValues() {
+                let selfValues: PathValue[] = [];
+                for (let value of values) {
+                    if (pathValueAuthority2.isSelfAuthority(value.path)) {
+                        selfValues.push(value);
+                    } else {
+                        let changed = writeValidStorage.setWriteValidStateValue(value);
+                        if (changed) {
+                            changes.push({ path: value.path, isValid: value.valid || false, time: value.time, isTransparent: value.isTransparent || false });
+                        }
+                    }
+                }
+                values = selfValues;
+            });
+        }
+
+        if (values.length === 0) return changes;
+
+        // Watch the value locks, for all SELF VALUES
+        //  (watches remote locks if necessary)
+        if (!alreadyWatching) {
+            lockWatcher.watchValueLocks(values);
+        }
+        let lockGroups = new Map<ReadLock[], PathValue[]>();
+        for (let value of values) {
+            if (value.lockCount === 0) {
+                validateLockCount(value);
+                let change: WriteState = {
+                    path: value.path,
+                    isValid: true,
+                    time: value.time,
+                    isTransparent: value.isTransparent || false
+                };
+                let changed = writeValidStorage.setWriteValidState(change, true);
+                if (changed) {
+                    changes.push(change);
+                }
+                continue;
+            }
+            let lockGroup = lockGroups.get(value.locks);
+            if (!lockGroup) {
+                lockGroup = [];
+                lockGroups.set(value.locks, lockGroup);
+            }
+            lockGroup.push(value);
+        }
+        for (let [locks, valueGroup] of lockGroups) {
+            if (valueGroup.length === 0) continue;
+
+            let rejected = false;
+            for (let lock of locks) {
+                if (!this.isLockValid(lock, now)) {
+                    rejected = true;
+
+                    // This is good... unless it happens A LOT. Then the app needs to change (unless it
+                    //  is a game, or something with realtime competition, in which case this is probably unavoidable).
+                    if ((!isCoreQuiet || !isNode()) && debugRejections || valueGroup.length > 0 && lock.endTime.version !== Number.MAX_SAFE_INTEGER && lock.startTime.version !== -2) {
+                        if (!isNode()) {
+                            debugger;
+                        }
+                        let timeToReject = now - valueGroup[0].time.time;
+                        console.log(red(`!!! VALUE REJECTED DUE TO USING MISSING / REJECTED READ!!!(rejected after ${timeToReject}ms at ${Date.now()})`));
+                        console.log(red(`${debugTime(valueGroup[0].time)} (write)`));
+                        console.log(red(`    rejected as the server could not find: `));
+                        if (lock.readIsTranparent) {
+                            console.log(red(`${debugTime(lock.startTime)} to ${debugTime(lock.endTime)} ${getPathFromStr(lock.path).join(".")} `));
+                        } else {
+                            console.log(red(`${debugTime(lock.startTime)} ${getPathFromStr(lock.path).join(".")} `));
+                        }
+                        if (lock.readIsTranparent) {
+                            console.log(red(`    (read was undefined, so presumably a value exists which the writer missed)`));
+                        }
+                        console.log(yellow(`Full list of writes rejected: `));
+                        for (let pathValue of valueGroup) {
+                            console.log(yellow(debugPathValue(pathValue)));
+                        }
+
+                        if (debugRejections) {
+                            debugbreak(2);
+                            debugger;
+                            this.isLockValid(lock, now);
+                            debugger;
+                        }
+                    }
+                    break;
+                }
+                let inRange = this.getWritesInRange(lock);
+                if (inRange?.length) {
+                    // If all the offending writes are undefined (or invalid), then we can ignore the contention
+                    if (lock.readIsTranparent && inRange.every(x => x.isTransparent || !x.isValid)) {
+                        continue;
+                    }
+                    rejected = true;
+                    // This is good... unless it happens A LOT. Then the app needs to change (unless it
+                    //  is a game, or something with realtime competition, in which case this is probably unavoidable).
+                    if (
+                        (!isCoreQuiet || !isNode())
+                        // This special version indicates clientside prediction (which SHOULD be rejected).
+                        && lock.endTime.version !== Number.MAX_SAFE_INTEGER
+                        || debugRejections
+                    ) {
+                        let changed = valueGroup.filter(x => x.valid);
+                        if (changed.length > 0) {
+                            let timeToReject = now - changed[0].time.time;
+                            console.log(red(`!!! LOCK CONTENTION FIXED VIA REJECTION OF VALUE!!!(rejected after ${timeToReject}ms)`));
+                            for (let pathValue of changed) {
+                                console.log(red(debugPathValue(pathValue)));
+                            }
+                            console.log(red(`    (write rejected due to original read not noticing value at: ${lock.path})`));
+                            console.log(red(`    (original read from: ${debugTime(lock.startTime)}`));
+                            console.log(red(`    (at time: ${debugTime(lock.endTime)}`));
+                            console.log(red(`    (current time: ${Date.now()})`));
+                            for (let lockFailed of inRange) {
+                                console.log(red(`    (conflict write at: ${debugTime(lockFailed.time)}`));
+                            }
+                            if (debugRejections) {
+                                debugbreak(2);
+                                debugger;
+                                this.getWritesInRange(lock);
+                                debugger;
+                            }
+                        }
+                    }
+                    break;
+                }
+            }
+            // NOTE: The if statement is equivalent to this one line, we just iterate so we can
+            //  have better debug info, and rejection metrics tracking.
+            //values.forEach(value => value.valid = !rejected);
+            if (rejected) {
+                for (let pathValue of valueGroup) {
+                    pathValue.valid = false;
+                    rejections.value++;
+                }
+            }
+
+            for (let value of valueGroup) {
+                let change: WriteState = {
+                    path: value.path,
+                    isValid: value.valid || false,
+                    time: value.time,
+                    isTransparent: value.isTransparent || false
+                };
+                let changed = writeValidStorage.setWriteValidState(change);
+                if (changed) {
+                    changes.push(change);
+                }
+            }
+        }
+
+        return changes;
+    }
+
+    public deleteRemovedPath(path: string) {
+        this.validStorage.delete(path);
+    }
+
+    private ensureGarbageCollectOldState = lazy(() => runInfinitePoll(MAX_CHANGE_AGE, () => {
+        // Clear all valid states after they are old enough. We will just tell anyone who asks that
+        //  they are valid, because... the must be if someone still cares about them (as either they have been
+        //  syncing for a long time, and so they must have never gotten an invalid message and it is too late
+        //  for that to change now, or... they are new, which means they will either be given old valid values,
+        //  or new values which may be invalid but are too new for us to garbage collect).
+        let garbageCollectTime = Date.now() - MAX_CHANGE_AGE;
+        for (let [key, values] of this.validStorage) {
+            values = values.filter(x => x.time.time > garbageCollectTime);
+            if (values.length === 0) {
+                this.validStorage.delete(key);
+            } else {
+                this.validStorage.set(key, values);
+            }
+        }
+    }));
+}
+export const writeValidStorage = new WriteValidStorage();
+
+let rejections = { value: 0 };
+export function internalGetRejections(): { value: number } {
+    return rejections;
+}
+export function internalTestResetRejections() {
+    rejections.value = 0;
+}
+
+class LockWatcher {
+    @measureFnc
+    public watchValueLocks(values: PathValue[]): void {
+        let { newRemoteLocks } = lockWatchDeduper.localWatchNewLocks({ values: values });
+        if (newRemoteLocks.size > 0) {
+            logErrors(this.watchRemoteLocks(newRemoteLocks));
+        }
+    }
+    private watchRemoteLocks = batchFunction(
+        { delay: "afterio" },
+        async (locksBatched: Set<ReadLock>[]) => {
+            let locks = new Set(locksBatched.flatMap(x => Array.from(x)));
+            let locksByAuthority = new Map<string, Set<ReadLock>>();
+            for (let lock of locks) {
+                if (pathValueAuthority2.isSelfAuthority(lock.path)) continue;
+
+                let authorityId = await pathValueAuthority2.getSingleReadNodePromise(lock.path);
+                // If they don't trust us, we just have to assume the value we read is correct.
+                if (!await isTrustedByNode(authorityId)) continue;
+                let locks = locksByAuthority.get(authorityId);
+                if (!locks) {
+                    locks = new Set();
+                    locksByAuthority.set(authorityId, locks);
+                }
+                locks.add(lock);
+            }
+
+            for (let [authorityId, locks] of locksByAuthority) {
+                this.watchLocksOnAuthority(authorityId, locks);
+            }
+        }
+    );
+    private watchLocksOnAuthority(authorityId: string, locks: Set<ReadLock>) {
+        if (!Array.from(locks).some(x => lockWatchDeduper.isLockStillWatched(x))) return;
+
+        let alreadyReconnected = false;
+        const reconnectWatches = () => {
+            if (alreadyReconnected) return;
+            alreadyReconnected = true;
+            if (!Array.from(locks).some(x => lockWatchDeduper.isLockStillWatched(x))) return;
+            logErrors(this.watchRemoteLocks(locks));
+        };
+        this.onNextDisconnectList(authorityId).add(reconnectWatches);
+        setTimeout(() => {
+            this.onNextDisconnectList(authorityId).delete(reconnectWatches);
+        }, MAX_CHANGE_AGE * 2);
+
+        let connection = PathValueController.nodes[authorityId].watchLockValid(
+            Array.from(locks.values())
+        );
+        logErrors(connection);
+        connection.catch(() => reconnectWatches());
+    }
+
+    private onNextDisconnectList = cache((nodeId: string) => {
+        let callbacks = new Set<() => void>();
+        SocketFunction.onNextDisconnect(nodeId, () => {
+            this.onNextDisconnectList.clear(nodeId);
+            for (let callback of callbacks) {
+                callback();
+            }
+        });
+        return callbacks;
+    });
+}
+export const lockWatcher = new LockWatcher();
+
+class LockToCallbackLookup {
+    // time.creatorId => time => callbacks
+    private validWatchers = registerResource("paths|validWatchers", new Map<number, Map<number, Set<WriteCallback>>>());
+
+    // path => sorted by endTime
+    private validRangeWatchers = registerResource("paths|validRangeWatchers", new Map<string, {
+        startTime: Time;
+        endTime: Time;
+        // Called back if there is a valid value between startTime (exclusive) and endTime (exclusive)
+        callback: WriteCallback;
+    }[]>());
+
+    /** Watchs both the startTime, and the range */
+    public watchLock(lock: ReadLock, callback: WriteCallback) {
+        this.ensureGarbageCollectLoop();
+        // Add valid watcher
+        if (lock.endTime.time > 0) {
+            const time = lock.startTime;
+            let maps = this.validWatchers.get(time.creatorId);
+            if (!maps) {
+                maps = new Map();
+                this.validWatchers.set(time.creatorId, maps);
+            }
+            let callbacksList = maps.get(time.time);
+            if (!callbacksList) {
+                callbacksList = new Set();
+                maps.set(time.time, callbacksList);
+            }
+            callbacksList.add(callback);
+        }
+        // Add valid range watcher
+        if (compareTime(lock.startTime, lock.endTime) !== 0) {
+            let { path, startTime, endTime } = lock;
+            let watchers = this.validRangeWatchers.get(path);
+            if (!watchers) {
+                watchers = [];
+                this.validRangeWatchers.set(path, watchers);
+            }
+            // Find the index to insert into, maintaining a sort by time
+            let index = -1;
+            for (let i = watchers.length - 1; i >= 0; i--) {
+                if (watchers[i].endTime.time < endTime.time) {
+                    index = i + 1;
+                    break;
+                }
+            }
+            if (index === -1) {
+                index = watchers.length;
+            }
+            watchers.splice(index, 0, { startTime, endTime, callback });
+        }
+    }
+
+    public garbageCollectOldWatchers() {
+        const disposeTime = Date.now() - MAX_CHANGE_AGE * 2;
+
+        for (let [path, times] of this.validRangeWatchers) {
+            let expiredIndex = times.findIndex(x => x.endTime.time < disposeTime);
+            if (expiredIndex >= 0) {
+                times.splice(expiredIndex, times.length - expiredIndex);
+            }
+        }
+
+        for (let [key, maps] of this.validWatchers) {
+            for (let time of maps.keys()) {
+                if (time < disposeTime) {
+                    maps.delete(time);
+                    if (maps.size === 0) {
+                        this.validWatchers.delete(key);
+                    }
+                }
+            }
+        }
+
+        for (let [path, watchers] of this.validRangeWatchers) {
+            for (let i = watchers.length - 1; i >= 0; i--) {
+                let watcher = watchers[i];
+                if (watcher.endTime.time < disposeTime) {
+                    watchers.splice(i, 1);
+                }
+            }
+            if (watchers.length === 0) {
+                this.validRangeWatchers.delete(path);
+            }
+        }
+    }
+
+    public getValidStateChangedTriggers(
+        remoteValue: WriteState
+    ): WriteCallback[] {
+        let allCallbacks: WriteCallback[] = [];
+        let validMaps = this.validWatchers.get(remoteValue.time.creatorId);
+        if (validMaps) {
+            let callbacks = validMaps.get(remoteValue.time.time);
+            if (callbacks) {
+                for (let callbackList of callbacks) {
+                    allCallbacks.push(callbackList);
+                }
+            }
+        }
+        let rangeWatchers = this.validRangeWatchers.get(remoteValue.path);
+        if (rangeWatchers) {
+            for (let watcher of rangeWatchers) {
+                if (
+                    // NOTE: This is the same comparison as in getWritesInRange
+                    // If the time is > startTime (exclusive, as startTime MUST exist, as we depend on it)
+                    compareTime(watcher.startTime, remoteValue.time) < 0
+                    // And < endTime (exclusive, as endTime is assumed to exist, as it is probably when we are writing!)
+                    && compareTime(remoteValue.time, watcher.endTime) < 0
+                ) {
+                    allCallbacks.push(watcher.callback);
+                }
+            }
+        }
+        return allCallbacks;
+    }
+
+    private ensureGarbageCollectLoop = lazy(() => {
+        runInfinitePoll(MAX_CHANGE_AGE, () => lockToCallback.garbageCollectOldWatchers());
+    });
+
+
+}
+export const lockToCallback = new LockToCallbackLookup();
+
+// NOTE: We assume there will be no equivalent but !== PathValues. We also assume
+//  there will be no equivalent but !== Locks. Fair assumptions, which save so
+//  much time it is probably worth maintaining them. AND, if they are invalidated,
+//  the only penalty is a memory leak, which isn't so bad (arguably having identical
+//  values with difference instances is a memory leak too).
+class LockWatchDeduper {
+    private watchedRemoteLocks = registerResource("paths|watchedRemoteLocks", new Map<ReadLock[], Set<PathValue>>());
+    private watchedLockFlat = registerResource("paths|watchedLockFlat", new Set<ReadLock>());
+
+    /** Calls validWatcher with any new readLocks, using the values as the callback.
+     *      SHOULD ONLY BE USED IF YOU ARE WATCHING THE LOCKS, aka, if you are
+     *          the authority on the values. Otherwise, you are NOT watching readLocks,
+     *          you are watching the valid state, via the actual authority of the paths.
+     */
+    public localWatchNewLocks(config: { values: PathValue[]; }): {
+        newRemoteLocks: Set<ReadLock>;
+    } {
+        this.ensureLockCleanupLoop();
+
+        let byLock = new Map<ReadLock[], PathValue[]>();
+        for (let value of config.values) {
+            validateLockCount(value);
+            if (value.lockCount === 0) continue;
+            let values = byLock.get(value.locks);
+            if (!values) {
+                values = [];
+                byLock.set(value.locks, values);
+            }
+            values.push(value);
+        }
+
+        let newLocks: ReadLock[] = [];
+        for (let value of config.values) {
+            validateLockCount(value);
+            if (value.lockCount === 0) continue;
+            let values = this.watchedRemoteLocks.get(value.locks);
+            if (!values) {
+                values = new Set();
+                this.watchedRemoteLocks.set(value.locks, values);
+                newLocks.push(...value.locks);
+            }
+            values.add(value);
+        }
+        for (let locks of byLock.keys()) {
+            for (let lock of locks) {
+                this.watchedLockFlat.add(lock);
+            }
+        }
+        let newRemoteLocks = new Set<ReadLock>();
+
+        for (let [locks, values] of byLock) {
+            for (let lock of locks) {
+                // NOTE: We watch LOCAL_DOMAIN paths, as they might depend on a remote value, which is invalidated.
+                //  This can happen for routing, where you click something like "go to main node", before
+                //  the remote node list is loaded. You will go to the wrong page, BUT, once the remote node list
+                //  is loaded, it should reject (and then, ideally we rerun the function clientside on detection
+                //  of the rejection).
+
+                if (!pathValueAuthority2.isSelfAuthority(lock.path)) {
+                    newRemoteLocks.add(lock);
+                }
+                // NOTE: I BELIEVE this is valid, even if we don't own the locks. We might have some
+                //  unnecessary cascading, but... if we depended on a value we will have it, and
+                //  if we are missing any range lock conflicts we will know about them due to
+                //  the remote locks. AND, any values we have are synced, so we don't need to worry about
+                //  any range lock conflicts themselves being rejected, missing that rejection, and so
+                //  incorrectly invalidating valid values that we just received.
+                lockToCallback.watchLock(lock, values);
+            }
+        }
+
+        return { newRemoteLocks };
+    }
+
+    private ensureLockCleanupLoop = lazy(() => {
+        runInfinitePoll(MAX_CHANGE_AGE, () => this.cleanupDeadLocks());
+    });
+
+    @measureFnc
+    private cleanupDeadLocks() {
+        let deadTime = Date.now() - MAX_CHANGE_AGE;
+        for (let [locks, values] of this.watchedRemoteLocks) {
+            let isDead = true;
+            for (let value of values) {
+                if (value.time.time > deadTime) {
+                    isDead = false;
+                    break;
+                }
+            }
+            if (isDead) {
+                for (let lock of locks) {
+                    this.watchedLockFlat.delete(lock);
+                }
+                this.watchedRemoteLocks.delete(locks);
+            }
+        }
+    }
+
+    public isLockStillWatched(lock: ReadLock): boolean {
+        return this.watchedLockFlat.has(lock);
+    }
+}
+export const lockWatchDeduper = new LockWatchDeduper();
+
+// Async import, as we are not allowed synchronous imports of values in
+//  higher tiers than us. We can't expose a callback, as nothing else
+//  can trigger the import of memoryValueAudit for all clients as consistently
+//  as the core.
+setImmediate(() => {
+    logErrors(import("../5-diagnostics/memoryValueAudit").then(x => x.startMemoryAuditLoop()));
+    logErrors(import("../5-diagnostics/diskValueAudit").then(x => x.startDiskAuditLoop()));
+    logErrors(import("../5-diagnostics/synchronousLagTracking").then(x => x.trackSynchronousLag()));
+});
+
+export { pathValueArchives };
+
+(globalThis as any).core = module.exports;
+(globalThis as any).SocketFunction = SocketFunction;