querysub 0.403.0 → 0.405.0

package/src/0-path-value-core/PathWatcher.ts

@@ -0,0 +1,534 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { cache } from "socket-function/src/caching";
+import { QueueLimited, keyByArray } from "socket-function/src/misc";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { debugNodeThread } from "../-c-identity/IdentityController";
+import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { registerResource } from "../diagnostics/trackResources";
+import { ignoreErrors } from "../errors";
+import { hack_stripPackedPath, getParentPathStr } from "../path";
+import { PathValueControllerBase } from "./PathValueController";
+import { PathRouter } from "./PathRouter";
+import { auditLog } from "./auditLogs";
+import { WatchConfig, authorityStorage, PathValue, NodeId, compareTime, isCoreQuiet, MAX_CHANGE_AGE, createMissingEpochValue } from "./pathValueCore";
+import { decodeParentFilter, matchesParentRangeFilter } from "./hackedPackedPathParentFiltering";
+
+
+// These are used so we can have our internal trigger lookup without watching them on remote values. 
+export function getInternalValueWatchPrefix() {
+    return "INTERNAL_VALUE_WATCH_PREFIX_";
+}
+
+// WATCH CASES
+// 1) getOwnNodeId() is used to trigger pathValueClientWatcher (no one else should be using it)
+// 2) Use other nodes to immediately proxy to the other nodes
+// 3) Use any string starting with the INTERNAL_VALUE_WATCH_PREFIX variable to watch them (remotely if required), but not trigger any callback. This works because the values are provided to the valid state computer before we receive them, so it doesn't require us to call them back (but it does require us to watch them remotely).
+//  - Valid state system uses this to watch values for locks
+type PathWatcherCallback = string;
+/*
+    PathWatcher is used if you want to either proxy watches to another server or if you just need the values, but don't need a callback triggered.
+        - If you want to watch with the callback, you should be using pathValueClientWatcher.
+*/
+class PathWatcher {
+    private watchers = registerResource("paths|PathWatcher.watchers", new Map<string, {
+        watchers: Set<PathWatcherCallback>;
+        requestedTime: number;
+        // When we last received an initial sync message, this isn't necessarily the first time we've ever synced it, just means this is when we started our current syncing. 
+        syncReceivedTime: number;
+    }>());
+    // If we don't limit the length, then this will memory leak. And if we only track
+    //  the values in watchers, then rolling key syncs will be lost.
+    // DOES NOT track local values, as they are used internally, but we aren't waiting for them to be synced.
+    private syncHistoryForHarvest = new QueueLimited<{ start: number; end: number; path: string; }>(1_000_000);
+    private syncHistoryForDebug = new QueueLimited<{ start: number; end: number; path: string; }>(1_000_000);
+
+    //private watchers = registerResource("paths|PathWatcher.watchers", new Map<string, Set<PathWatcherCallback>>());
+
+    // realPath => packedPath => watcher => { depth, start, end }[]
+    private parentWatchers = registerResource("paths|parentWatchers", new Map<string,
+        // packedPath
+        Map<string, {
+            watchers: Set<PathWatcherCallback>;
+        }>
+    >());
+    private watchersToPaths = registerResource("paths|watchersToPaths", new Map<PathWatcherCallback, {
+        paths: Set<string>;
+        parents: Set<string>;
+        fullHistory?: boolean;
+    }>());
+
+    /** Automatically watches the remote if it's a remote path.
+     *  Automatically unwatches the remote when the paths no longer have any callbacks.
+     *  - Only stores callbacks in a Set, as strings, so unique ones are deduped, but the first unwatch will unwatch them all.
+     *   */
+    @measureFnc
+    public watchPath(config: WatchConfig & {
+        nodeId: PathWatcherCallback;
+
+        debugName?: string;
+        noInitialTrigger?: boolean;
+        /** Force trigger, even if we are already watching the values. */
+        initialTrigger?: boolean;
+    }) {
+        this.ensureUnwatchingOnDisconnect(config.nodeId);
+        if (config.nodeId.startsWith(getInternalValueWatchPrefix()) && (!config.noInitialTrigger || config.initialTrigger)) {
+            require("debugbreak")(2);
+            debugger;
+            // No initial trigger because if it's a remote value, then we won't have it. If it's a local value, then we've already processed it in valid state computers, there's no reason to trigger it again. 
+            throw new Error("Internal value watches (INTERNAL_VALUE_WATCH_PREFIX) cannot have initial triggers");
+        }
+
+        let newPathsWatched = new Set<string>();
+        let newParentsWatched = new Set<string>();
+
+        let time = Date.now();
+
+        for (let path of config.paths) {
+            let watchers = this.watchers.get(path);
+            if (!watchers) {
+                watchers = {
+                    watchers: new Set(),
+                    requestedTime: time,
+                    syncReceivedTime: 0,
+                };
+                this.watchers.set(path, watchers);
+            }
+            if (watchers.watchers.has(config.nodeId)) continue;
+            newPathsWatched.add(path);
+            watchers.watchers.add(config.nodeId);
+
+            let watchObj = this.watchersToPaths.get(config.nodeId);
+            if (!watchObj) {
+                watchObj = { paths: new Set(), parents: new Set() };
+                this.watchersToPaths.set(config.nodeId, watchObj);
+            }
+            watchObj.paths.add(path);
+            if (config.fullHistory) {
+                watchObj.fullHistory = true;
+            }
+        }
+        for (let path of config.parentPaths) {
+            let basePath = hack_stripPackedPath(path);
+            let watchersObj = this.parentWatchers.get(basePath);
+            if (!watchersObj) {
+                watchersObj = new Map();
+                this.parentWatchers.set(basePath, watchersObj);
+            }
+            let obj = watchersObj.get(path);
+            if (!obj) {
+                obj = { watchers: new Set() };
+                watchersObj.set(path, obj);
+            }
+            if (obj.watchers.has(config.nodeId)) continue;
+            obj.watchers.add(config.nodeId);
+
+            newParentsWatched.add(path);
+
+            let watchObj = this.watchersToPaths.get(config.nodeId);
+            if (!watchObj) {
+                watchObj = { paths: new Set(), parents: new Set() };
+                this.watchersToPaths.set(config.nodeId, watchObj);
+            }
+            watchObj.parents.add(path);
+        }
+
+        if (newPathsWatched.size > 0 || newParentsWatched.size > 0) {
+            if (isOwnNodeId(config.nodeId)) {
+                for (let path of newPathsWatched) {
+                    auditLog("new local WATCH VALUE", { path });
+                }
+                for (let path of newParentsWatched) {
+                    auditLog("new local WATCH PARENT", { path });
+                }
+            } else {
+                for (let path of newPathsWatched) {
+                    auditLog("new non-local WATCH VALUE", {
+                        path,
+                        watcher: config.nodeId,
+                        sourceNodeThreadId: debugNodeThread(config.nodeId),
+                    });
+                }
+                for (let path of newParentsWatched) {
+                    auditLog("new non-local WATCH PARENT", { path, watcher: config.nodeId });
+                }
+            }
+            console.info(`New PathValue watches`, {
+                newPathsWatched: newPathsWatched.size,
+                newParentsWatched: newParentsWatched.size,
+            });
+        }
+        if (newPathsWatched.size > 0 || newParentsWatched.size > 0) {
+            if (!this.remoteWatchCallback) throw new Error("remoteWatchCallback not set by the pathWatcher");
+            let newRemotePaths = Array.from(newPathsWatched).filter(x => !PathRouter.isSelfAuthority(x));
+            let newRemoteParentPaths = Array.from(newParentsWatched).filter(x => !PathRouter.isSelfAuthority(x));
+            if (newRemotePaths.length > 0 || newRemoteParentPaths.length > 0) {
+                this.remoteWatchCallback({
+                    paths: newRemotePaths,
+                    parentPaths: newRemoteParentPaths,
+                });
+            }
+        }
+
+        // Treat everything as a new watch
+        if (config.initialTrigger) {
+            newPathsWatched = new Set(config.paths);
+            newParentsWatched = new Set(config.parentPaths);
+        }
+
+        if (!config.noInitialTrigger) {
+            this.triggerValuesChanged({
+                valuesChanged: new Set(),
+                initialTriggers: { values: newPathsWatched, parentPaths: newParentsWatched },
+            });
+        }
+    }
+    // NOTE: Automatically unwatches from remoteWatcher on all paths that are fully unwatched
+    @measureFnc
+    public unwatchPath(config: WatchConfig & { callback: PathWatcherCallback; }) {
+        const { callback } = config;
+
+        let fullyUnwatched: WatchConfig = {
+            paths: [],
+            parentPaths: [],
+        };
+
+        let pathsWatched = this.watchersToPaths.get(callback);
+
+        for (let path of config.parentPaths) {
+            if (pathsWatched) {
+                pathsWatched.parents.delete(path);
+            }
+
+            let unpackedPath = hack_stripPackedPath(path);
+
+            let watchersObj = this.parentWatchers.get(unpackedPath);
+            if (!watchersObj) continue;
+            let obj = watchersObj.get(path);
+            if (!obj) continue;
+            obj.watchers.delete(callback);
+
+            if (isOwnNodeId(callback)) {
+                auditLog("local UNWATCH PARENT", { path });
+            } else {
+                auditLog("non-local UNWATCH PARENT", { path, watcher: callback });
+            }
+
+            if (obj.watchers.size === 0) {
+                watchersObj.delete(path);
+                this.parentWatchers.delete(path);
+                fullyUnwatched.parentPaths.push(path);
+                authorityStorage.markParentPathAsUnwatched(path);
+
+                if (watchersObj.size === 0) {
+                    this.parentWatchers.delete(unpackedPath);
+
+                    let childPaths = authorityStorage.getPathsFromParent(unpackedPath);
+                    // Destroy values that now have no watchers (no value watchers, and now no parent watcher
+                    if (childPaths) {
+                        for (let childPath of childPaths) {
+                            if (!this.watchers.has(childPath)) {
+                                authorityStorage.destroyPath(childPath);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // NOTE: Unwatch parents, then values, as parent paths might keep alive value path watches.
+        for (let path of config.paths) {
+            if (pathsWatched) {
+                pathsWatched.paths.delete(path);
+            }
+
+            let watchers = this.watchers.get(path);
+            if (!watchers) continue;
+            watchers.watchers.delete(callback);
+
+            if (isOwnNodeId(callback)) {
+                auditLog("local UNWATCH VALUE", { path });
+            } else {
+                auditLog("non-local UNWATCH VALUE", { path, watcher: callback });
+            }
+
+            if (watchers.watchers.size === 0) {
+                this.watchers.delete(path);
+
+                let parentPath = getParentPathStr(path);
+
+                fullyUnwatched.paths.push(path);
+                // NOTE: If the parent is being watched, don't destroy the value.
+                //  - The fact that we only do the check here does mean that if the path is unwatched
+                //      first and then later the parent path is unwatched we will fail to properly destroy
+                //      the path. However, in practice, either both are unwatched at the same time, or more
+                //      likely, the value watch only goes away because it's made redundant by the parent watch.
+                if (!this.parentWatchers.has(parentPath)) {
+                    authorityStorage.destroyPath(path);
+                }
+            }
+        }
+
+        if (fullyUnwatched.paths.length > 0 || fullyUnwatched.parentPaths.length > 0) {
+            console.info(`Unwatched PathValue watches`, {
+                unwatchedPaths: fullyUnwatched.paths.length,
+                unwatchedParents: fullyUnwatched.parentPaths.length,
+            });
+            if (!this.unwatchedCallbacks) throw new Error("unwatchedCallbacks not set by the RemoteWatcher");
+            this.unwatchedCallbacks(fullyUnwatched);
+        }
+    }
+
+    public unwatchEventPaths(paths: Set<string>) {
+        for (let path of paths) {
+            let watchers = this.watchers.get(path);
+            if (watchers) {
+                this.watchers.delete(path);
+                for (let watcher of watchers.watchers) {
+                    let watchObj = this.watchersToPaths.get(watcher);
+                    if (watchObj) {
+                        watchObj.paths.delete(path);
+                    }
+                }
+            }
+            let parentWatchers = this.parentWatchers.get(path);
+            if (parentWatchers) {
+                this.parentWatchers.delete(path);
+                for (let { watchers } of parentWatchers.values()) {
+                    for (let watcher of watchers) {
+                        let watchObj = this.watchersToPaths.get(watcher);
+                        if (watchObj) {
+                            watchObj.parents.delete(path);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /** You can provide any values that you want us to trigger. You can also provide initial triggers, which can send values by themselves, depending on what type of watcher it is. */
+    @measureFnc
+    public triggerValuesChanged(config: {
+        // IMPORTANT! The place that triggers this likely can only be our committing code as it knows what is valid or not, and explicitly calculates what's valid or not. If the valid states on these path values are wrong, we'll broadcast them and everyone will have the wrong valid states.
+        // ALSO, These path values need to be ingested first.
+        valuesChanged: Set<PathValue>;
+
+        // We mutate initialTriggers
+        initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+
+        onlyTriggerNodeId?: string;
+    }) {
+        let { valuesChanged, initialTriggers, onlyTriggerNodeId } = config;
+
+        let changedPerCallbacks: Map<PathWatcherCallback, {
+            values: Set<PathValue>;
+            initialTriggers: { values: Set<string>; parentPaths: Set<string> };
+        }> = new Map();
+        let time = Date.now();
+
+        let triggerPaths = new Set<string>();
+        for (let value of valuesChanged) {
+            triggerPaths.add(value.path);
+        }
+        for (let path of initialTriggers?.values ?? []) {
+            triggerPaths.add(path);
+        }
+
+        let valuePerPath = keyByArray(Array.from(valuesChanged), x => x.path);
+
+        // initialTriggers.parentPaths => triggerPaths
+        for (let parentPath of initialTriggers.parentPaths) {
+            let valuePaths = authorityStorage.getPathsFromParent(parentPath);
+            for (let valuePath of valuePaths || []) {
+                triggerPaths.add(valuePath);
+                config.initialTriggers.values.add(valuePath);
+            }
+
+            let latestParentWatches = this.parentWatchers.get(hack_stripPackedPath(parentPath));
+            if (!latestParentWatches) continue;
+            for (let { watchers } of latestParentWatches.values()) {
+                for (let watcher of watchers) {
+                    let changes = changedPerCallbacks.get(watcher);
+                    if (!changes) {
+                        changes = { values: new Set(), initialTriggers: { values: new Set(), parentPaths: new Set() } };
+                        changedPerCallbacks.set(watcher, changes);
+                    }
+                    changes.initialTriggers.parentPaths.add(parentPath);
+                }
+            }
+        }
+
+        // triggerPaths => changedPerCallbacks + value histories of initial triggered values
+        for (let path of triggerPaths) {
+            let values = valuePerPath.get(path) ?? [];
+            let latestWatches = this.watchers.get(path);
+            if (latestWatches && !latestWatches.syncReceivedTime && !PathRouter.isLocalPath(path) && initialTriggers?.values.has(path)) {
+                latestWatches.syncReceivedTime = time;
+                let obj = { start: latestWatches.requestedTime, end: time, path };
+                this.syncHistoryForHarvest.push(obj);
+                this.syncHistoryForDebug.push(obj);
+            }
+
+            // Important. We can't do the initial trigger if we aren't synced. If we're the authority, this doesn't matter. This is only important for proxy watchers. 
+            let isInitialTrigger = !!initialTriggers?.values.has(path) && authorityStorage.isSynced(path);
+            const triggerNodeChanged = (watcher: NodeId) => {
+                if (onlyTriggerNodeId && watcher !== onlyTriggerNodeId) return;
+                let changes = changedPerCallbacks.get(watcher);
+                if (!changes) {
+                    changes = { values: new Set(), initialTriggers: { values: new Set(), parentPaths: new Set() } };
+                    changedPerCallbacks.set(watcher, changes);
+                }
+                // Always send the explicit triggers (new values)
+                // - This will result in some redundant values being sent if these aren't the latest values (and fullHistory is off). However, it shouldn't make a difference in speed as it's very unusual for the values to be out of order. And certain things, such as the function runner, wants all the values, and there's debugging cases when having all the values will be useful as well. 
+                for (let value of values) {
+                    changes.values.add(value);
+                }
+                if (isInitialTrigger) {
+                    let watcherObj = this.watchersToPaths.get(watcher);
+                    let fullHistory = watcherObj?.fullHistory;
+                    if (fullHistory) {
+                        let history = authorityStorage.getValuePlusHistory(path);
+                        for (let historicalValue of history) {
+                            changes.values.add(historicalValue);
+                        }
+                        if (history.length === 0) {
+                            changes.values.add(createMissingEpochValue(path));
+                        }
+                    } else {
+                        // NOTE: We won't be given a value if it isn't synced, which will be never because we check if we're synced above. However, the type system doesn't know that. 
+                        let value = authorityStorage.getValueAtTime(path);
+                        if (value) {
+                            changes.values.add(value);
+                        }
+                    }
+                }
+            };
+
+            if (latestWatches) {
+                for (let watch of latestWatches.watchers) {
+                    triggerNodeChanged(watch);
+                }
+            }
+
+            let parentPath = getParentPathStr(path);
+            let latestParentWatches = this.parentWatchers.get(parentPath);
+            if (latestParentWatches) {
+                for (let [packedPath, { watchers }] of latestParentWatches) {
+                    if (!matchesParentRangeFilter({ parentPath, fullPath: path, packedPath })) continue;
+                    for (let watch of watchers) {
+                        triggerNodeChanged(watch);
+                    }
+                }
+            }
+        }
+
+
+        for (let [watch, { values, initialTriggers }] of changedPerCallbacks) {
+            this.triggerLatestWatcher(watch, Array.from(values), initialTriggers);
+        }
+    }
+
+    public isWatching(path: string): boolean {
+        return this.watchers.has(path) || this.parentWatchers.has(getParentPathStr(path));
+    }
+
+    @measureFnc
+    private triggerLatestWatcher(
+        watcher: PathWatcherCallback,
+        changes: PathValue[],
+        initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
+    ) {
+        // NOTE: The receivers will sort these anyway, however it makes the logs cleaner if we send them in the correct order.
+        changes.sort((a, b) => compareTime(a.time, b.time));
+        if (isOwnNodeId(watcher)) {
+            if (!this.localTriggerCallback) throw new Error("localTriggerCallback not set by the pathValueClientWatcher");
+            this.localTriggerCallback(changes, initialTriggers?.parentPaths ?? new Set());
+        } else if (watcher.startsWith(getInternalValueWatchPrefix())) {
+            // The valid state computer will get the value before we even get it, so it doesn't need to be told to trigger. It triggers in all values.
+        } else {
+            if (!isCoreQuiet) {
+                console.log(`(${Date.now()}) Sending values to client: ${changes.length} (${watcher})`);
+            }
+            this.ensureUnwatchingOnDisconnect(watcher);
+            ignoreErrors((async () => {
+                await PathValueControllerBase.sendValues({
+                    nodeId: watcher,
+                    pathValues: changes,
+                    initialTriggers: initialTriggers,
+                });
+            })());
+        }
+    }
+
+    private ensureUnwatchingOnDisconnect = cache((nodeId: string) => {
+        if (isOwnNodeId(nodeId)) return;
+        if (nodeId.startsWith(getInternalValueWatchPrefix())) return;
+        SocketFunction.onNextDisconnect(nodeId, async () => {
+            this.ensureUnwatchingOnDisconnect.clear(nodeId);
+
+            // Wait while, so Querysub doesn't thrash when clients connect and disconnect
+            //  TODO: We should be smarter about our wait time, waiting less if we have a lot of resource
+            //      pressure, and more (potentially forever), if we don't have much resource pressure
+            //      (memory, cpu, and network).
+            setTimeout(() => {
+                let watches = this.watchersToPaths.get(nodeId);
+                this.watchersToPaths.delete(nodeId);
+                if (watches) {
+                    this.unwatchPath({ paths: Array.from(watches.paths), parentPaths: Array.from(watches.parents), callback: nodeId });
+                }
+            }, MAX_CHANGE_AGE);
+        });
+    });
+
+    private localTriggerCallback: ((changes: PathValue[], parentPaths: Set<string>) => void) | undefined;
+    public setLocalTriggerCallback(callback: (changes: PathValue[], parentPaths: Set<string>) => void) {
+        if (this.localTriggerCallback) throw new Error("localTriggerCallback already set. It should only be set once, by the pathValueClientWatcher");
+        this.localTriggerCallback = callback;
+    }
+
+    private unwatchedCallbacks: ((config: WatchConfig) => void) | undefined;
+    public setUnwatchCallback(callback: (config: WatchConfig) => void) {
+        if (this.unwatchedCallbacks) throw new Error("unwatchedCallbacks already set. It should only be set once, by the RemoteWatcher");
+        this.unwatchedCallbacks = callback;
+    }
+
+    private remoteWatchCallback: ((config: WatchConfig) => void) | undefined;
+    public setWatchRemoteCallback(callback: (config: WatchConfig) => void) {
+        if (this.remoteWatchCallback) throw new Error("remoteWatchCallback already set. It should only be set once, by the RemoteWatcher");
+        this.remoteWatchCallback = callback;
+    }
+
+
+    public getAllParentWatches() {
+        return Array.from(this.parentWatchers.keys());
+    }
+
+    public debug_harvestSyncTimes(): { start: number; end: number; path: string; }[] {
+        let times = this.syncHistoryForHarvest.getAllUnordered();
+        this.syncHistoryForHarvest.reset();
+        return times;
+    }
+
+    public debug_getSyncHistory(): { start: number; end: number; path: string; }[] {
+        return this.syncHistoryForDebug.getAllUnordered();
+    }
+
+    public getWatcherStats() {
+        let localPathCount = 0;
+        let remotePathCount = 0;
+
+        for (let [path] of this.watchers) {
+            if (PathRouter.isLocalPath(path)) {
+                localPathCount++;
+            } else {
+                remotePathCount++;
+            }
+        }
+
+        return {
+            localPathCount,
+            remotePathCount,
+        };
+    }
+}
+export const pathWatcher = new PathWatcher();
+(globalThis as any).pathWatcher = pathWatcher;

package/src/0-path-value-core/ShardPrefixes.ts

@@ -0,0 +1,31 @@
+
+/*
+    By default, sharding works by hashing the hash to get the route. This technically works, however, it means if you have an object with ten properties, you could easily end up talking to 10 different servers.
+    
+    And so we have a list of prefixes, and if we match a prefix, then we hash less of the path (We accidentally hashed the child key under the prefix, but that's for a different reason related to making sure parallel lookups use the same server).
+
+    The prefix needs to be detailed enough so that everything doesn't use the same server.
+
+    After a PathServerValue starts, it can't change its prefixes, as if the routing changes it makes it too difficult for clients to reason about who they should be synchronizing with.
+*/
+
+import { nestArchives } from "../-a-archives/archives";
+import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
+import { archiveJSONT } from "../-a-archives/archivesJSONT";
+import { getDomain } from "../config";
+
+type PrefixObj = {
+    prefixes: string[];
+};
+
+let prefixes = archiveJSONT<PrefixObj>(() => nestArchives("shard-prefixes/", getArchivesBackblaze(getDomain())));
+// We don't actually need a collection, we only need a single value. However, the collection interface is a lot more convenient than manually dealing with backplays files. 
+const key = "all";
+
+export async function getShardPrefixes(): Promise<string[]> {
+    return (await prefixes.get(key))?.prefixes ?? [];
+}
+
+export async function setShardPrefixes(_prefixes: string[]) {
+    await prefixes.set(key, { prefixes: _prefixes });
+}