querysub 0.2.0

package/src/1-path-client/RemoteWatcher.ts

@@ -0,0 +1,558 @@
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { batchFunction, delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
+import { isNode, nextId } from "socket-function/src/misc";
+import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
+import { isOwnNodeId, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { PathValueController, pathValueCommitter } from "../0-path-value-core/PathValueController";
+import { WatchConfig, authorityStorage, decodeParentFilter, matchesParentRangeFilter, pathWatcher } from "../0-path-value-core/pathValueCore";
+import { ActionsHistory } from "../diagnostics/ActionsHistory";
+import { registerResource } from "../diagnostics/trackResources";
+import { logErrors } from "../errors";
+import { ClientWatcher } from "./pathValueClientWatcher";
+import { blue, green, yellow } from "socket-function/src/formatting/logColors";
+import { appendToPathStr, getParentPathStr, getPathDepth, getPathIndexAssert, getPathSuffix, hack_getPackedPathSuffix, hack_setPackedPathSuffix } from "../path";
+import { isEmpty } from "../misc";
+import debugbreak from "debugbreak";
+import { isDevDebugbreak } from "../config";
+import { debugLog } from "../0-path-value-core/debugLogs";
+
+export class RemoteWatcher {
+    public static DEBUG = false;
+
+    public static REMOTE_WATCH_FUNCTION = (config: WatchConfig, authorityId: string) => PathValueController.nodes[authorityId].watchLatest(config);
+    public static REMOTE_UNWATCH_FUNCTION = (config: WatchConfig, authorityId: string) => PathValueController.nodes[authorityId].unwatchLatest(config);
+
+    // path => nodeId
+    //  isOwnNodeId(nodeId) means we are watching it locally
+    private remoteWatchPaths = registerResource("paths|remoteWatchPaths", new Map<string, string>());
+    private remoteWatchParents = registerResource("paths|remoteWatchParents", new Map<string, {
+        nodesId: Map<string, {
+            start: number;
+            end: number;
+        }>;
+        fullNodeCount: number;
+    }>());
+
+    private disconnectedPaths = registerResource("paths|disconnectedPaths", new Set<string>());
+    private disconnectedParents = registerResource("paths|disconnectedParents", new Set<string>());
+
+    private reconnectCache = cache((authorityId: string) => {
+        SocketFunction.onNextDisconnect(authorityId, () => {
+            logErrors((async () => {
+                // NOTE: While we have no remote unwatcher we leave the values in our cache. They might be stale
+                //  for a bit (which might result in our writes being rejected), but... hopefully we find a new
+                //  server to watch really fast! (Otherwise... if there is no authority... stale data is better
+                //  than no data).
+                this.reconnectCache.clear(authorityId);
+                let paths: string[] = [];
+                for (let [path, remoteAuthorityId] of this.remoteWatchPaths.entries()) {
+                    if (remoteAuthorityId === authorityId) {
+                        paths.push(path);
+                        this.remoteWatchPaths.delete(path);
+                    }
+                }
+                let parentPaths: string[] = [];
+                for (let [path, watchObj] of this.remoteWatchParents.entries()) {
+                    if (typeof watchObj.nodesId === "string") {
+                        if (watchObj.nodesId === authorityId) {
+                            this.remoteWatchParents.delete(path);
+                            parentPaths.push(path);
+                        }
+                    } else {
+                        if (watchObj.nodesId.has(authorityId)) {
+                            watchObj.nodesId.delete(authorityId);
+                            if (isEmpty(watchObj.nodesId)) {
+                                this.remoteWatchParents.delete(path);
+                            }
+                            parentPaths.push(path);
+                        }
+                    }
+                }
+                if (paths.length === 0 && parentPaths.length === 0) return;
+
+                for (let path of paths) {
+                    this.disconnectedPaths.add(path);
+                }
+                for (let path of parentPaths) {
+                    this.disconnectedParents.add(path);
+                }
+
+                // NOTE: We keep around the old authorities in routing (for a few minutes), so if a new node doesn't start,
+                //  we will match the disconnected node again. This means we have a few minutes to start another server.
+                // TODO: Retry instead of throwing
+                console.log(yellow(`Trying to find new authority for disconnected watches ${paths.length} paths and ${parentPaths.length} parent paths`));
+                logErrors(this.tryToReconnectNow());
+            })());
+        });
+    });
+
+    // NOTE: Lower numbers mean more lag on disconnected values, but... faster reconnection. AND... if a node is disconnected,
+    //      the chances the server will be useless is high, so... lag is probably fine in exchange for faster
+    //      reconnection.
+    private reconnectLoop = runInfinitePoll(10000, () => this.tryToReconnectNow());
+    private async tryToReconnectNow() {
+        if (!this.disconnectedPaths.size && !this.disconnectedParents.size) return;
+
+        // We HAVE to watch synchronously, so there is no gap where a path can be removed from disconnectedPaths,
+        //      and we can trigger it to be re-added
+        this.internalWatchLatest({
+            paths: Array.from(this.disconnectedPaths),
+            parentPaths: Array.from(this.disconnectedParents),
+            tryReconnect: true,
+        });
+    }
+
+
+    public debugIsWatchingPath(path: string) {
+        // HACK: If there is no read node... pretend we are watching it. Because... for now, this is a known
+        //  and the code calling this function isn't looking for that specific issue.
+        return this.remoteWatchPaths.has(path) || !pathValueAuthority2.getSingleReadNodeSync(path);
+    }
+    public debugIsWatchingParentPath(path: string) {
+        return this.remoteWatchParents.has(path) || !pathValueAuthority2.getSingleReadNodeSync(path);
+    }
+
+    public watchLatest(config: WatchConfig & { debugName?: string }) {
+        logErrors(this.watchLatestPromise(config));
+    }
+    // NOTE: This is private, as who wants to wait for watch to finish?
+    //      1) The value won't be ready when it finishes.
+    //      2) If it errors out, they can't do anything to fix the error
+    //      3) We retry internally anyways
+    private watchLatestPromise(config: WatchConfig & { debugName?: string }) {
+        // NOTE: If none of the values are remote... early out. This has been profile to save a bit of time,
+        //  mostly due to avoiding the async call.
+        if (config.parentPaths.length === 0 && config.paths.every(x => pathValueAuthority2.isSelfAuthority(x))) {
+            return;
+        }
+        return this.watchLatestBase(config);
+    }
+
+    /** Returns [] if we aren't remotely syncing the path. */
+    public getMultiNodesForParent(path: string): Map<string, unknown> | undefined {
+        let watchObj = this.remoteWatchParents.get(path);
+        return watchObj?.nodesId;
+    }
+
+    private watchUnwatchSerial = runInSerial((name: string, fnc: () => Promise<unknown>) =>
+        measureBlock(fnc, name)
+    );
+
+    /** NOTE: We dedupe duplicate watches in watchLatest. */
+    private async watchLatestBase(config: WatchConfig & { debugName?: string }) {
+        await this.watchUnwatchSerial("watchLatestBase", async () => {
+            await pathValueAuthority2.waitUntilRoutingIsReady();
+
+            this.internalWatchLatest(config);
+        });
+    }
+    // IMPORTANT! This function is expected to be synchronous, due to how tryToReconnectNow works.
+    @measureFnc
+    private internalWatchLatest(config: WatchConfig & {
+        debugName?: string;
+        tryReconnect?: boolean;
+    }) {
+        let watchesPerAuthority = new Map<string, { paths: string[]; parentPaths: string[] }>();
+
+        let unwatchParentsPerAuthority = new Map<string, Set<string>>();
+
+        let newDisconnectPaths = 0;
+        let newDisconnectParents = 0;
+        let foundPaths = 0;
+        let foundParentPaths = 0;
+        measureBlock(() => {
+            for (let path of config.paths) {
+                // IMPORTANT! We have to await, otherwise we might see the paths are watched, skip them
+                //  find another path which has an authority, then go to watch that... which causes
+                //  us to circumvent earlier watchers, causing us to watch out of order, causing reads
+                //  to come back out of order, causing watching functions to trigger out of order!
+                let remoteAuthorityId = this.remoteWatchPaths.get(path);
+                if (remoteAuthorityId !== undefined) continue;
+
+                let authorityId = pathValueAuthority2.getSingleReadNodeSync(path);
+                if (!authorityId) {
+                    if (!this.disconnectedPaths.has(path)) {
+                        newDisconnectPaths++;
+                        this.disconnectedPaths.add(path);
+                    }
+                    continue;
+                }
+                this.disconnectedPaths.delete(path);
+                foundPaths++;
+
+                this.remoteWatchPaths.set(path, authorityId);
+                let paths = watchesPerAuthority.get(authorityId);
+                if (!paths) {
+                    paths = { paths: [], parentPaths: [] };
+                    watchesPerAuthority.set(authorityId, paths);
+                }
+                paths.paths.push(path);
+            }
+
+            for (let path of config.parentPaths) {
+                let watchObj = this.remoteWatchParents.get(path);
+                if (
+                    watchObj
+                    && watchObj.nodesId.size === watchObj.fullNodeCount
+                    && !config.tryReconnect
+                ) continue;
+
+                let { nodes } = pathValueAuthority2.getChildReadNodes(path, {
+                    // Pass existing connected node ids as preferred
+                    preferredNodeIds: watchObj?.nodesId
+                });
+
+                let rangeStart = 0;
+                let rangeEnd = 1;
+
+                let nodesToAdd = new Map<string, { start: number; end: number }>();
+                let range = decodeParentFilter(path);
+                if (range) {
+                    rangeStart = range.start;
+                    rangeEnd = range.end;
+                }
+
+                {
+                    let curStart = 0;
+                    for (let node of nodes) {
+                        let r = node.range;
+                        if (r.start > curStart) {
+                            console.warn(`There is a gap in the range of nodes for path ${path} values from from ${curStart} to ${r.start} cannot be found in any nodes.`);
+                            curStart = r.start;
+                        }
+                        let nextEnd = Math.min(r.end, rangeEnd);
+                        // Ignore values with no overlap (mostly trailing values)
+                        if (nextEnd <= curStart) continue;
+
+                        nodesToAdd.set(node.nodeId, { start: curStart, end: nextEnd });
+                        curStart = nextEnd;
+                    }
+                }
+
+                let fullRangeCovered = true;
+                {
+                    let lastValue = rangeStart;
+                    for (let node of nodes) {
+                        if (node.range.start > lastValue) {
+                            fullRangeCovered = false;
+                            break;
+                        }
+                        lastValue = node.range.end;
+                    }
+                    if (lastValue < rangeEnd) {
+                        fullRangeCovered = false;
+                    }
+                }
+                if (!fullRangeCovered) {
+                    if (!this.disconnectedParents.has(path)) {
+                        newDisconnectParents++;
+                        this.disconnectedParents.add(path);
+                    }
+                } else {
+                    this.disconnectedParents.delete(path);
+                    foundParentPaths++;
+                }
+
+                if (watchObj) {
+                    for (let nodeId of watchObj.nodesId.keys()) {
+                        if (!nodesToAdd.has(nodeId)) {
+                            let parentPaths = unwatchParentsPerAuthority.get(nodeId);
+                            if (!parentPaths) {
+                                parentPaths = new Set();
+                                unwatchParentsPerAuthority.set(nodeId, parentPaths);
+                            }
+                            parentPaths.add(path);
+                            watchObj.nodesId.delete(nodeId);
+                        }
+                    }
+                }
+
+                if (!watchObj) {
+                    watchObj = { nodesId: new Map(), fullNodeCount: 0 };
+                    this.remoteWatchParents.set(path, watchObj);
+                }
+                watchObj.fullNodeCount = nodes.length;
+                for (let [nodeId, range] of nodesToAdd) {
+                    watchObj.nodesId.set(nodeId, range);
+                }
+
+                for (let authorityId of nodesToAdd.keys()) {
+                    let paths = watchesPerAuthority.get(authorityId);
+                    if (!paths) {
+                        paths = { paths: [], parentPaths: [] };
+                        watchesPerAuthority.set(authorityId, paths);
+                    }
+                    paths.parentPaths.push(path);
+                }
+            }
+        }, `watchLatest|getSingleReadNode`);
+
+        if (newDisconnectPaths > 0 || newDisconnectParents > 0) {
+            console.log(yellow(`Some paths have no authority. We will search for an authority periodically until we find an authority. New missing ${newDisconnectPaths} paths and ${newDisconnectParents} parent paths, total missing ${this.disconnectedPaths.size} paths and ${this.disconnectedParents.size} parent paths.`));
+            let first10 = Array.from(this.disconnectedPaths).slice(0, 10);
+            for (let path of first10) {
+                console.log(`\t${path}`);
+            }
+            if (this.disconnectedPaths.size > 10) {
+                console.log(`\t... and ${this.disconnectedPaths.size - 10} more paths`);
+            }
+        }
+        if (config.tryReconnect) {
+            if (foundPaths || foundParentPaths) {
+                console.log(green(`Reconnected ${foundPaths} paths and ${foundParentPaths} parent paths.`));
+            }
+            if (this.disconnectedPaths.size > 0 || this.disconnectedParents.size > 0) {
+                console.log(yellow(`We still have ${this.disconnectedPaths.size} paths and ${this.disconnectedParents.size} parent paths disconnected.`));
+            }
+        }
+
+        for (let [authorityId, { paths, parentPaths }] of watchesPerAuthority.entries()) {
+            // Only remote watch remotes!
+            if (isOwnNodeId(authorityId)) continue;
+
+            if (RemoteWatcher.DEBUG) {
+                console.log(`${blue("Watching")} ${paths.length} paths and ${parentPaths.length} parent paths on ${authorityId}\n\t    authority: ${blue(nodePathAuthority.getAuthorityPaths(authorityId)?.map(x => nodePathAuthority.getArchiveDirectory(x)).join(" | ") || "")}`);
+                if (parentPaths.length > 0) {
+                    console.log(`\t    parentPaths:`);
+                    for (let path of parentPaths) {
+                        console.log(`\t    ${path}`);
+                    }
+                }
+            }
+
+            if (isDevDebugbreak()) {
+                for (let path of paths) {
+                    debugLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
+                }
+                for (let path of parentPaths) {
+                    debugLog("remoteWatcher outer WATCH PARENT", { path, remoteNodeId: authorityId });
+                }
+            }
+
+            this.reconnectCache(authorityId);
+            // Don't throw, as the reconnectCache will eventually resolve errors and watch successfully
+            logErrors(this.watchLatestRemote({
+                authorityId,
+                paths,
+                parentPaths,
+                debugName: config.debugName
+            }));
+        }
+
+        // No throttling for unwatches, so we don't have race conditions?
+        for (let [authorityId, parentPaths] of unwatchParentsPerAuthority.entries()) {
+            if (isOwnNodeId(authorityId)) continue;
+            logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths: [], parentPaths: Array.from(parentPaths) }, authorityId));
+        }
+    }
+
+    public getChildPatchWatchChecker(config: {
+        parentPath: string;
+        nodeId: string;
+    }): {
+        isWatchedByNodeId(childPath: string): boolean;
+    } {
+        let remoteNodes = this.remoteWatchParents.get(config.parentPath);
+        if (!remoteNodes) return { isWatchedByNodeId: () => false };
+        const range = remoteNodes.nodesId.get(config.nodeId);
+        if (!range) return { isWatchedByNodeId: () => false };
+        return {
+            isWatchedByNodeId(childPath) {
+                return matchesParentRangeFilter({
+                    parentPath: config.parentPath,
+                    fullPath: childPath,
+                    start: range.start,
+                    end: range.end,
+                });
+            },
+        };
+    }
+
+    private watchLatestRemote = batchFunction(
+        { delay: 10, throttleWindow: isNode() ? 500 : undefined },
+        async (batched: (WatchConfig & { authorityId: string; debugName?: string })[]) => {
+            await this.watchUnwatchSerial("watchLatestRemote", async () => {
+                let byAuthority = new Map<string, { paths: Set<string>; parentPaths: Set<string>; }>();
+                for (let { authorityId, paths, parentPaths } of batched) {
+                    let pathsPerAuthority = byAuthority.get(authorityId);
+                    if (!pathsPerAuthority) {
+                        pathsPerAuthority = { paths: new Set(), parentPaths: new Set() };
+                        byAuthority.set(authorityId, pathsPerAuthority);
+                    }
+
+                    for (let path of paths) {
+                        if (this.remoteWatchPaths.get(path) !== authorityId) continue;
+                        pathsPerAuthority.paths.add(path);
+                    }
+                    for (let path of parentPaths) {
+                        let watchObj = this.remoteWatchParents.get(path);
+                        if (!watchObj) continue;
+                        if (!watchObj.nodesId.has(authorityId)) continue;
+                        pathsPerAuthority.parentPaths.add(path);
+                    }
+                }
+                for (let [authorityId, { paths, parentPaths }] of byAuthority) {
+                    if (isDevDebugbreak()) {
+                        for (let path of paths) {
+                            debugLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
+                        }
+                        for (let path of parentPaths) {
+                            debugLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
+                        }
+                    }
+                    // NOTE: We log watches here because we want to log batched watches
+                    ActionsHistory.OnNewWatches({ newPathsWatched: paths, newParentsWatched: parentPaths, debugName: batched[0].debugName, authorityId });
+                    let config: WatchConfig = {
+                        paths: Array.from(paths),
+                        parentPaths: Array.from(parentPaths),
+                    };
+                    logErrors(RemoteWatcher.REMOTE_WATCH_FUNCTION(config, authorityId));
+                }
+            });
+        }
+    );
+
+    /** NOTE: We do not count the number of watches, so... only unwatch if everything is done watching a path! */
+
+    public unwatchLatest(config: WatchConfig) {
+        // IMPORTANT! This serial is important, otherwise there is a race condition between starting to unwatch,
+        //      and unwatching, which can result in marking a path as unwatched when it is watched.
+        //      1) We want to unwatch, and call unwatchLatest
+        //      2) There is a delay in unwatchLatest
+        //      3) We want to watch again, and do, and receive a new value
+        //      4) unwatchLatest runs, causing us to mark the path as unsynced
+        //      5) The user of the watch runs, see's the value is unsynced, and so waits,
+        //          but it never receives the value.
+        //  We ALSO could just check if the path is watched here, but... that is extra work, and... the path
+        //      from here to where we unwatch is pretty short (PathWatcher.unwatchPath => hook => RemoteWatcher.unwatchLatest),
+        //      so I don't imagine any more bugs... hopefully.
+        logErrors(this.watchUnwatchSerial("unwatchLatest", async () => {
+            this.unwatchLatestBase(config);
+        }));
+    }
+    @measureFnc
+    private unwatchLatestBase(config: WatchConfig) {
+        let unwatchesPerAuthority = new Map<string, WatchConfig>();
+
+        const innerPathRemove = (path: string, remoteAuthorityId: string | undefined) => {
+            if (remoteAuthorityId === undefined) return;
+            this.remoteWatchPaths.delete(path);
+            this.disconnectedPaths.delete(path);
+            let paths = unwatchesPerAuthority.get(remoteAuthorityId);
+            if (!paths) {
+                paths = { paths: [], parentPaths: [] };
+                unwatchesPerAuthority.set(remoteAuthorityId, paths);
+            }
+            paths.paths.push(path);
+        };
+        const innerParentPathRemove = (path: string, remoteAuthorityId: string | undefined) => {
+            if (remoteAuthorityId === undefined) return;
+            this.remoteWatchParents.delete(path);
+            this.disconnectedParents.delete(path);
+            let paths = unwatchesPerAuthority.get(remoteAuthorityId);
+            if (!paths) {
+                paths = { paths: [], parentPaths: [] };
+                unwatchesPerAuthority.set(remoteAuthorityId, paths);
+            }
+            paths.parentPaths.push(path);
+        };
+
+        for (let path of config.paths) {
+            let remoteAuthorityId = this.remoteWatchPaths.get(path);
+            innerPathRemove(path, remoteAuthorityId);
+        }
+        for (let path of config.parentPaths) {
+            let watchObj = this.remoteWatchParents.get(path);
+            if (!watchObj) continue;
+            for (let authorityId of watchObj.nodesId.keys()) {
+                innerParentPathRemove(path, authorityId);
+            }
+            this.remoteWatchParents.delete(path);
+        }
+        for (let [authorityIdBase, { paths, parentPaths }] of unwatchesPerAuthority.entries()) {
+            if (isDevDebugbreak()) {
+                for (let path of paths) {
+                    debugLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
+                }
+                for (let path of parentPaths) {
+                    debugLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
+                }
+            }
+            if (!isOwnNodeId(authorityIdBase)) {
+                logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths, parentPaths }, authorityIdBase));
+            }
+        }
+    }
+
+    @measureFnc
+    public unwatchEventPaths(config: WatchConfig) {
+        // NOTE: We HAVE to tell the remote to unwatch... because, it know not have the full PathValue,
+        //  and so not know it is an event path (we could have receive it from another node), so...
+        //  it might NOT know it should dispose it!
+        this.unwatchLatest(config);
+    }
+
+    public getExistingWatchRemoteNodeId(path: string) {
+        return this.remoteWatchPaths.get(path);
+    }
+    public getAllRemoteWatchedNodeIds() {
+        return Array.from(new Set(this.remoteWatchPaths.values()));
+    }
+
+    public async refreshAllWatches(authorityNodeId: string) {
+        await pathValueAuthority2.waitUntilRoutingIsReady();
+        await this.watchUnwatchSerial("refreshAllWatches", async () => {
+            // Unwatch all paths, reset all remoteWatchPaths, and then watch all paths again SYNCHRONOUSLY,
+            //      so there is no time we aren't watching the paths.
+            //  - AND parent paths as well (at the same time)
+            // NOTE: This might result in the same authority being used, which is fine. The unwatch + watch
+            //      should run in serial (due to how connections work), and result in being sent a new value.
+            //      It might also result in a different authority being used, which is fine, and can resolve
+            //      cases where an authority is in a broken state.
+
+            let pathsToWatch = new Set<string>();
+            let parentPathsToWatch = new Set<string>();
+            for (let [path, nodeId] of this.remoteWatchPaths) {
+                if (nodeId === authorityNodeId) {
+                    pathsToWatch.add(path);
+                }
+            }
+            // Delete after, so we don't break the iteration
+            for (let path of pathsToWatch) {
+                this.remoteWatchPaths.delete(path);
+            }
+            for (let [path, watchObj] of this.remoteWatchParents) {
+                if (watchObj.nodesId.has(authorityNodeId)) {
+                    watchObj.nodesId.delete(authorityNodeId);
+                    parentPathsToWatch.add(path);
+                }
+            }
+            let paths = Array.from(pathsToWatch);
+            let parentPaths = Array.from(parentPathsToWatch);
+
+            logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths, parentPaths }, authorityNodeId));
+            this.internalWatchLatest({
+                paths: Array.from(pathsToWatch),
+                parentPaths: Array.from(parentPathsToWatch),
+                debugName: "refreshAllWatches",
+            });
+        });
+    }
+
+    public getWatchReason(path: string): "direct" | "parent" | undefined {
+        if (this.remoteWatchPaths.has(path)) return "direct";
+        if (this.remoteWatchParents.has(getParentPathStr(path))) return "parent";
+        return undefined;
+    }
+}
+
+export const remoteWatcher = new RemoteWatcher();
+(globalThis as any).remoteWatcher = remoteWatcher;
+setImmediate(() => {
+    pathWatcher.watchUnwatched(config => remoteWatcher.unwatchLatest(config));
+    authorityStorage.setGetMultiNodesForParent(path => remoteWatcher.getMultiNodesForParent(path));
+    pathValueCommitter.setGetRemoteWatchNodeId(path => remoteWatcher.getExistingWatchRemoteNodeId(path));
+});