querysub 0.403.0 → 0.405.0

package/src/0-path-value-core/startupAuthority.ts

@@ -0,0 +1,74 @@
+import { delay } from "socket-function/src/batching";
+import { DISK_FLUSH_INTERVAL } from "../diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants";
+import { authorityLookup } from "./AuthorityLookup";
+import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { PathValueControllerBase, pathValueCommitter } from "./PathValueController";
+import { validStateComputer } from "./ValidStateComputer";
+import { MAX_TIME_UNTIL_DISK_FLUSH, authorityStorage, pathValueArchives } from "./pathValueCore";
+import { blue, green } from "socket-function/src/formatting/logColors";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+
+const MAX_LOAD_RETRIES = 3;
+export async function startupAuthority(spec: AuthoritySpec) {
+    let startTime = Date.now();
+    console.log(`Becoming authority for`, { spec });
+
+    // NOTE: This will broadcast all the other servers about our spec, and so all other servers will start giving us live writes.
+    await authorityLookup.setOurSpec(spec);
+
+    let readStartTime = Date.now() - MAX_TIME_UNTIL_DISK_FLUSH;
+    let readEndTime = Number.MAX_SAFE_INTEGER;
+
+    let sources = await PathRouter.getAuthoritySources({ target: spec });
+    console.log(`Found sources when finding spec, count: ${sources.length}`, { spec, sources });
+    async function loadFromSourceBase(source: AuthoritySpec) {
+        console.log(blue(`Loading values from source ${source.nodeId}`), { nodeId: source.nodeId, spec: source, readStartTime, readEndTime });
+        let values = await PathValueControllerBase.getInitialValues({
+            nodeId: source.nodeId,
+            spec: source,
+            startTime: readStartTime,
+            endTime: readEndTime,
+        });
+        console.log(blue(`Finished loading values from source ${source.nodeId}, found ${formatNumber(values.length)} values`), { nodeId: source.nodeId, spec: source, values: values.length });
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: values,
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
+        });
+        console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: source, values: values.length });
+    }
+    async function loadFromSource(source: AuthoritySpec, retries = 3) {
+        try {
+            await loadFromSourceBase(source);
+            return;
+        } catch (error: any) {
+            console.error(`Failed to load from source ${source.nodeId}`, { error: error.stack });
+            await delay(1000);
+            if (retries > 0) {
+                let sources = await PathRouter.getAuthoritySources({ target: source });
+                await Promise.all(sources.map(x => loadFromSource(x, retries - 1)));
+            } else {
+                console.error(`Failed to load from source ${source.nodeId} after ${MAX_LOAD_RETRIES} retries`);
+            }
+        }
+    }
+
+    let sourcePromises = sources.map(x => loadFromSource(x));
+
+    // Oh, we need to load snapshots as well... of course
+    let diskPromise = (async () => {
+        let snapshot = await pathValueArchives.loadValues(spec);
+        let flat = Object.values(snapshot.values).flat();
+        console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec, values: flat.length });
+        // NOTE: If it's on disk, it can't be rejected, so there's no point in passing it through our validStateComputer (and passing it through is VERY slow).
+        authorityStorage.ingestValues(flat, { doNotArchive: true });
+        console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec, values: flat.length });
+    })();
+
+    await Promise.all([...sourcePromises, diskPromise]);
+
+    await authorityLookup.setIsReady();
+    let duration = Date.now() - startTime;
+    console.log(green(`Finished becoming authority, took ${formatTime(duration)}`), { spec });
+}

package/src/1-path-client/RemoteWatcher.ts

@@ -1,28 +1,31 @@
 
 import { SocketFunction } from "socket-function/SocketFunction";
-import { batchFunction, delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { batchFunction, runInSerial, runInfinitePoll } from "socket-function/src/batching";
 import { cache } from "socket-function/src/caching";
-import { isNode, nextId } from "socket-function/src/misc";
+import { isNode } from "socket-function/src/misc";
 import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
-import { isOwnNodeId, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { LOCAL_DOMAIN_PATH, nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
-import { PathValueController, pathValueCommitter } from "../0-path-value-core/PathValueController";
-import { WatchConfig, authorityStorage, decodeParentFilter, matchesParentRangeFilter, pathWatcher } from "../0-path-value-core/pathValueCore";
+import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { PathValueController } from "../0-path-value-core/PathValueController";
+import { PathRouter } from "../0-path-value-core/PathRouter";
+import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
+import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
 import { registerResource } from "../diagnostics/trackResources";
 import { logErrors } from "../errors";
-import { ClientWatcher } from "./pathValueClientWatcher";
-import { blue, green, yellow } from "socket-function/src/formatting/logColors";
-import { appendToPathStr, getParentPathStr, getPathDepth, getPathIndexAssert, getPathSuffix, hack_getPackedPathSuffix, hack_setPackedPathSuffix } from "../path";
+import { green, yellow } from "socket-function/src/formatting/logColors";
+import { getParentPathStr } from "../path";
 import { isEmpty } from "../misc";
-import debugbreak from "debugbreak";
 import { isDevDebugbreak } from "../config";
 import { auditLog, isDebugLogEnabled } from "../0-path-value-core/auditLogs";
 import { debugNodeThread } from "../-c-identity/IdentityController";
+import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
+import { decodeParentFilter, encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
+setImmediate(() => import("../4-querysub/Querysub"));
 
 // NOTE: In some cases this can half our the PathValues sent. AND, it can reduce our sync requests by N, and as those are uploads those are event more expensive. However, it also complicates other code, requiring not just checking if a value is synced, but also the parent path. We should try to make this stable, but worst case scenario, we can set this to false, and it will make our synchronization a lot easier to verify.
 const STOP_KEYS_DOUBLE_SENDS = true;
 
+// NOTE: If a parent watch is broken up between multiple nodes, we generally watch everything on all those nodes and then filter when we receive the data. This isn't efficient, and we should probably change it. However, in practice, it's probably fine, as it's unlikely for the watches to be broken up to a much finer granularity than the network (it would require very strange partially overlapping sharding cases which no reasonable sharding setup would ever satisfy).
 export class RemoteWatcher {
     public static DEBUG = false;
 
@@ -32,7 +35,11 @@ export class RemoteWatcher {
     // path => nodeId
     //  isOwnNodeId(nodeId) means we are watching it locally
     private remoteWatchPaths = registerResource("paths|remoteWatchPaths", new Map<string, string>());
+    // inputPath => 
     private remoteWatchParents = registerResource("paths|remoteWatchParents", new Map<string, {
+        useFullPathHash: boolean;
+        // Might vary from the input path if useFullPathHash is true (and the path is a hacked path, and so which has a range restriction).
+        watchedPath: string;
         nodesId: Map<string, {
             start: number;
             end: number;
@@ -116,12 +123,13 @@ export class RemoteWatcher {
     public debugIsWatchingPath(path: string) {
         // HACK: If there is no read node... pretend we are watching it. Because... for now, this is a known
         //  and the code calling this function isn't looking for that specific issue.
-        return this.remoteWatchPaths.has(path) || !pathValueAuthority2.getSingleReadNodeSync(path);
+        return this.remoteWatchPaths.has(path) || !PathRouter.getReadyAuthority(path);
     }
     public debugIsWatchingParentPath(path: string) {
-        return this.remoteWatchParents.has(path) || !pathValueAuthority2.getSingleReadNodeSync(path);
+        return this.remoteWatchParents.has(path) || !PathRouter.getReadyAuthority(path);
     }
 
+    /** @deprecated If you want to watch something, call pathWatcher.watchPath instead. */
     public watchLatest(config: WatchConfig & { debugName?: string }) {
         logErrors(this.watchLatestPromise(config));
     }
@@ -131,10 +139,9 @@ export class RemoteWatcher {
     //      2) If it errors out, they can't do anything to fix the error
     //      3) We retry internally anyways
     private watchLatestPromise(config: WatchConfig & { debugName?: string }) {
-        // NOTE: If none of the values are remote... early out. This has been profiled to save a bit of time,
-        //  mostly due to avoiding the async call.
+        // NOTE: If none of the values are remote... early out. This has been profiled to save a bit of time, mostly due to avoiding the async call.
         if (config.parentPaths.length === 0) {
-            let isSelf = measureBlock(() => config.paths.every(x => pathValueAuthority2.isSelfAuthority(x)), "RemoteWatcher()|watchLatestPromise|isSelfCheck");
+            let isSelf = measureBlock(() => config.paths.every(x => PathRouter.isSelfAuthority(x)), "RemoteWatcher()|watchLatestPromise|isSelfCheck");
             if (isSelf) {
                 return;
             }
@@ -155,7 +162,7 @@ export class RemoteWatcher {
     /** NOTE: We dedupe duplicate watches in watchLatest. */
     private async watchLatestBase(config: WatchConfig & { debugName?: string }) {
         await this.watchUnwatchSerial("watchLatestBase", async () => {
-            await pathValueAuthority2.waitUntilRoutingIsReady();
+            await authorityLookup.startSyncing();
 
             this.internalWatchLatest(config);
         });
@@ -181,7 +188,7 @@ export class RemoteWatcher {
         measureBlock(() => {
             for (let path of config.paths) {
                 // NOTE: We weren't ignoring local paths here for a while, but... I'm pretty sure we should, to save time.
-                if (path.startsWith(LOCAL_DOMAIN_PATH)) continue;
+                if (PathRouter.isLocalPath(path)) continue;
 
                 // IMPORTANT! We have to await, otherwise we might see the paths are watched, skip them
                 //  find another path which has an authority, then go to watch that... which causes
@@ -201,10 +208,11 @@ export class RemoteWatcher {
                     continue;
                 }
 
-                let authorityId = pathValueAuthority2.getSingleReadNodeSync(path);
+                let authorityId = PathRouter.getReadyAuthority(path)?.nodeId;
                 if (!authorityId) {
                     if (totalMissingPaths === 0) {
-                        let authorities = pathValueAuthority2.debug_getAllAuthorities();
+                        let authorities = authorityLookup.getTopologySync();
+                        authorityId = PathRouter.getReadyAuthority(path)?.nodeId;
                         console.warn(`Missing authority for path ${path}`, { authorities });
                     }
                     totalMissingPaths++;
@@ -228,29 +236,53 @@ export class RemoteWatcher {
                 paths.paths.push(path);
             }
 
-            for (let path of config.parentPaths) {
+            for (let originalPath of config.parentPaths) {
                 // NOTE: We weren't ignoring local paths here for a while, but... I'm pretty sure we should, to save time.
-                if (path.startsWith(LOCAL_DOMAIN_PATH)) continue;
-                let watchObj = this.remoteWatchParents.get(path);
+                if (PathRouter.isLocalPath(originalPath)) continue;
+                let watchObj = this.remoteWatchParents.get(originalPath);
                 if (
                     watchObj
                     && watchObj.nodesId.size === watchObj.fullNodeCount
                     && !config.tryReconnect
                 ) continue;
 
-                let { nodes } = pathValueAuthority2.getChildReadNodes(path, {
+                let { useFullPathHash, nodes } = PathRouter.getChildReadNodes(originalPath, {
                     // Pass existing connected node ids as preferred
-                    preferredNodeIds: watchObj?.nodesId
+                    preferredNodeIds: watchObj && Array.from(watchObj.nodesId.keys())
                 });
+                let watchedPath = originalPath;
+                useFullPathHash = !!useFullPathHash;
+                let decodedParentFilter = decodeParentFilter(originalPath);
+                if (useFullPathHash && decodedParentFilter) {
+                    watchedPath = encodeParentFilter({
+                        path: decodedParentFilter.path,
+                        startFraction: decodedParentFilter.start,
+                        endFraction: decodedParentFilter.end,
+                        useFullPathHash: true,
+                    });
+                }
+
+                // Unwatch the old watchedPath
+                if (watchObj && watchObj.watchedPath !== watchedPath) {
+                    for (let nodeId of watchObj.nodesId.keys()) {
+                        let parentPaths = unwatchParentsPerAuthority.get(nodeId);
+                        if (!parentPaths) {
+                            parentPaths = new Set();
+                            unwatchParentsPerAuthority.set(nodeId, parentPaths);
+                        }
+                        parentPaths.add(watchObj.watchedPath);
+                    }
+
+                    watchObj = undefined;
+                }
 
                 let rangeStart = 0;
                 let rangeEnd = 1;
 
                 let nodesToAdd = new Map<string, { start: number; end: number }>();
-                let range = decodeParentFilter(path);
-                if (range) {
-                    rangeStart = range.start;
-                    rangeEnd = range.end;
+                if (decodedParentFilter) {
+                    rangeStart = decodedParentFilter.start;
+                    rangeEnd = decodedParentFilter.end;
                 }
 
                 {
@@ -258,7 +290,7 @@ export class RemoteWatcher {
                     for (let node of nodes) {
                         let r = node.range;
                         if (r.start > curStart) {
-                            console.warn(`There is a gap in the range of nodes for path ${path} values from from ${curStart} to ${r.start} cannot be found in any nodes.`);
+                            console.warn(`There is a gap in the range of nodes for path ${originalPath} values from from ${curStart} to ${r.start} cannot be found in any nodes.`);
                             curStart = r.start;
                         }
                         let nextEnd = Math.min(r.end, rangeEnd);
@@ -285,15 +317,16 @@ export class RemoteWatcher {
                     }
                 }
                 if (!fullRangeCovered) {
-                    if (!this.disconnectedParents.has(path)) {
+                    if (!this.disconnectedParents.has(originalPath)) {
                         newDisconnectParents++;
-                        this.disconnectedParents.add(path);
+                        this.disconnectedParents.add(originalPath);
                     }
                 } else {
-                    this.disconnectedParents.delete(path);
+                    this.disconnectedParents.delete(originalPath);
                     foundParentPaths++;
                 }
 
+                // Unwatch nodes that have changed
                 if (watchObj) {
                     for (let nodeId of watchObj.nodesId.keys()) {
                         if (!nodesToAdd.has(nodeId)) {
@@ -302,28 +335,29 @@ export class RemoteWatcher {
                                 parentPaths = new Set();
                                 unwatchParentsPerAuthority.set(nodeId, parentPaths);
                             }
-                            parentPaths.add(path);
+                            parentPaths.add(watchedPath);
                             watchObj.nodesId.delete(nodeId);
                         }
                     }
                 }
 
                 if (!watchObj) {
-                    watchObj = { nodesId: new Map(), fullNodeCount: 0, suppressedWatches: new Set() };
-                    this.remoteWatchParents.set(path, watchObj);
+                    watchObj = { nodesId: new Map(), fullNodeCount: 0, suppressedWatches: new Set(), watchedPath, useFullPathHash };
+                    this.remoteWatchParents.set(watchedPath, watchObj);
                 }
                 watchObj.fullNodeCount = nodes.length;
                 for (let [nodeId, range] of nodesToAdd) {
                     watchObj.nodesId.set(nodeId, range);
                 }
 
+                // NOTE: We watch every node with the full request. If our nodes are sharded well, this should be fine. They won't have that much data to give us, and we'll ignore it when we receive it if it's more than we want. 
                 for (let authorityId of nodesToAdd.keys()) {
                     let paths = watchesPerAuthority.get(authorityId);
                     if (!paths) {
                         paths = { paths: [], parentPaths: [] };
                         watchesPerAuthority.set(authorityId, paths);
                     }
-                    paths.parentPaths.push(path);
+                    paths.parentPaths.push(watchedPath);
                 }
             }
         }, `watchLatest|getSingleReadNode`);
@@ -351,22 +385,12 @@ export class RemoteWatcher {
             // Only remote watch remotes!
             if (isOwnNodeId(authorityId)) continue;
 
-            if (RemoteWatcher.DEBUG) {
-                console.log(`${blue("Watching")} ${paths.length} paths and ${parentPaths.length} parent paths on ${authorityId}\n\t    authority: ${blue(nodePathAuthority.getAuthorityPaths(authorityId)?.map(x => nodePathAuthority.getArchiveDirectory(x)).join(" | ") || "")}`);
-                if (parentPaths.length > 0) {
-                    console.log(`\t    parentPaths:`);
-                    for (let path of parentPaths) {
-                        console.log(`\t    ${path}`);
-                    }
-                }
-            }
-
             if (isDevDebugbreak()) {
                 for (let path of paths) {
                     auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
                 }
                 for (let path of parentPaths) {
-                    auditLog("remoteWatcher outer WATCH PARENT", { path, remoteNodeId: authorityId });
+                    auditLog("remoteWatcher outer PARENT WATCH", { path, remoteNodeId: authorityId });
                 }
             }
 
@@ -398,31 +422,10 @@ export class RemoteWatcher {
         }
     }
 
-    public getChildPatchWatchChecker(config: {
-        parentPath: string;
-        nodeId: string;
-    }): {
-        isWatchedByNodeId(childPath: string): boolean;
-    } {
-        let remoteNodes = this.remoteWatchParents.get(config.parentPath);
-        if (!remoteNodes) return { isWatchedByNodeId: () => false };
-        const range = remoteNodes.nodesId.get(config.nodeId);
-        if (!range) return { isWatchedByNodeId: () => false };
-        return {
-            isWatchedByNodeId(childPath) {
-                return matchesParentRangeFilter({
-                    parentPath: config.parentPath,
-                    fullPath: childPath,
-                    start: range.start,
-                    end: range.end,
-                });
-            },
-        };
-    }
-
     private watchLatestRemote = batchFunction(
         { delay: 10, throttleWindow: isNode() ? 500 : undefined },
         async (batched: (WatchConfig & { authorityId: string; debugName?: string })[]) => {
+            const { Querysub } = await import("../4-querysub/Querysub");
             await this.watchUnwatchSerial("watchLatestRemote", async () => {
                 let byAuthority = new Map<string, { paths: Set<string>; parentPaths: Set<string>; }>();
                 for (let { authorityId, paths, parentPaths } of batched) {
@@ -457,6 +460,7 @@ export class RemoteWatcher {
                     let config: WatchConfig = {
                         paths: Array.from(paths),
                         parentPaths: Array.from(parentPaths),
+                        fullHistory: Querysub.SEND_FULL_HISTORY_ON_INITIAL_SYNC,
                     };
                     if (isDebugLogEnabled()) {
                         for (let path of paths) {
@@ -472,8 +476,7 @@ export class RemoteWatcher {
         }
     );
 
-    /** NOTE: We do not count the number of watches, so... only unwatch if everything is done watching a path! */
-
+    /** @deprecated Call pathWatcher.unwatchPath instead. */
     public unwatchLatest(config: WatchConfig) {
         // IMPORTANT! This serial is important, otherwise there is a race condition between starting to unwatch,
         //      and unwatching, which can result in marking a path as unwatched when it is watched.
@@ -573,13 +576,16 @@ export class RemoteWatcher {
         let parentWatchObj = this.remoteWatchParents.get(parentPath);
         if (parentWatchObj) {
             for (let [nodeId, obj] of parentWatchObj.nodesId) {
-                if (matchesParentRangeFilter({
-                    parentPath,
-                    fullPath: path,
-                    start: obj.start,
-                    end: obj.end,
-                })) {
-                    return nodeId;
+                if (parentWatchObj.useFullPathHash) {
+                    let route = PathRouter.getSingleKeyRoute(path);
+                    if (obj.start <= route && route < obj.end) {
+                        return nodeId;
+                    }
+                } else {
+                    let route = PathRouter.getRouteChildKey(path);
+                    if (obj.start <= route && route < obj.end) {
+                        return nodeId;
+                    }
                 }
             }
             // NOTE: I think it is a bug if we hit here. There shouldn't be a parent, but have us not match it?
@@ -591,8 +597,10 @@ export class RemoteWatcher {
         return Array.from(new Set(this.remoteWatchPaths.values()));
     }
 
+
+
     public async refreshAllWatches(authorityNodeId: string) {
-        await pathValueAuthority2.waitUntilRoutingIsReady();
+        await authorityLookup.startSyncing();
         await this.watchUnwatchSerial("refreshAllWatches", async () => {
             // Unwatch all paths, reset all remoteWatchPaths, and then watch all paths again SYNCHRONOUSLY,
             //      so there is no time we aren't watching the paths.
@@ -640,8 +648,8 @@ export class RemoteWatcher {
 
 export const remoteWatcher = new RemoteWatcher();
 (globalThis as any).remoteWatcher = remoteWatcher;
-setImmediate(() => {
-    pathWatcher.watchUnwatched(config => remoteWatcher.unwatchLatest(config));
+void Promise.resolve().finally(() => {
+    pathWatcher.setUnwatchCallback(config => remoteWatcher.unwatchLatest(config));
+    pathWatcher.setWatchRemoteCallback(config => remoteWatcher.watchLatest(config));
     authorityStorage.setGetMultiNodesForParent(path => remoteWatcher.getMultiNodesForParent(path));
-    pathValueCommitter.setGetRemoteWatchNodeId(path => remoteWatcher.getExistingWatchRemoteNodeId(path));
 });