querysub 0.508.0 → 0.509.0

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -0,0 +1,290 @@
+/*
+    Tracks the FunctionRunners in the cluster: which networks they listen on, their shard ranges, whether they are public, how long they have been up, and the latency of talking to them. Also tracks call durations (overall, and per network), in 5 minute buckets, for the last hour.
+
+    This is all PER QUERYSUB SERVER, in memory (the latency and call times depend on the node making the calls) — it is NEVER written to the database. Clients get their querysub server's index from the edge node config at startup (so it is available immediately, with no calls), and then refresh it by polling QuerysubController.getFunctionRunnerIndex, using it to automatically select the best network for their function calls.
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute, timeInSecond, sort } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { isClient } from "../config2";
+import { t } from "../2-proxy/schema2";
+import { createLocalSchema } from "./schemaHelpers";
+import { Querysub } from "./Querysub";
+import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
+import { URLParam } from "../library-components/URLParam";
+import { timeoutToUndefinedSilent } from "../errors";
+import { spreadCallsOverTime } from "../misc";
+import { formatTime } from "socket-function/src/formatting/format";
+import type { DebugFunctionShardInfo } from "../3-path-functions/PathFunctionRunner";
+import type { EdgeNodeConfig } from "../4-deploy/edgeNodes";
+import { isPublic } from "../config";
+
+const POLL_INTERVAL = timeInMinute;
+const INDEX_POLL_INTERVAL = timeInMinute * 5;
+// Forget runners we haven't been able to reach for this long
+const NODE_EXPIRY_TIME = POLL_INTERVAL * 5;
+// Rolling latency average size. Small, so latency changes show up quickly.
+const LATENCY_SAMPLE_LIMIT = 20;
+// Only prefer runners that have been up at least this long
+const MIN_UP_TIME = timeInMinute * 5;
+// Only weight by call time once a network has at least this much total recorded call time
+const MIN_TOTAL_CALL_TIME_FOR_WEIGHTING = timeInSecond * 60;
+// Added to the average call time when weighting, so small call time differences don't give near infinite priority (2s vs 0s becomes 3 / 1 priority, instead of infinite)
+const CALL_TIME_WEIGHT_BASE = 1000;
+// If every runner on a network will be shut down within this window, the network is dying and we switch off of it.
+const NETWORK_DYING_WINDOW = timeInMinute * 2;
+
+// The forced network is a URL parameter, so it survives reloads and is shareable / obvious in the URL.
+export const forcedNetworkURL = new URLParam("network", "");
+
+export type FunctionRunnerNodeInfo = {
+    nodeId: string;
+    entryPoint: string;
+    startupTime: number;
+    lastSeen: number;
+    isPublic: boolean;
+    networks: string[];
+    shards: DebugFunctionShardInfo[];
+    averageLatency: number;
+    latencySampleCount: number;
+    // Set once the runner has been told it will shut down (scheduled switchover)
+    scheduledShutdownTime?: number;
+};
+export type FunctionRunnerIndex = {
+    nodes: FunctionRunnerNodeInfo[];
+    // Stats over ALL function calls in the last hour
+    callStats: FunctionStatsSummary;
+    // network => stats over the last hour
+    networkStats: { [network: string]: FunctionStatsSummary };
+};
+export type NetworkSelectionInfo = {
+    network: string;
+    nodeCount: number;
+    isPublic: boolean;
+    upLongEnough: boolean;
+    // Every runner on this network is scheduled to shut down within NETWORK_DYING_WINDOW
+    dying: boolean;
+    averageLatency: number;
+    averageCallTime: number;
+    score: number;
+};
+
+let nodeInfos = new Map<string, FunctionRunnerNodeInfo>();
+// runner nodeId => that runner's stats over our own calls (network => summary, already summarized by the runner)
+let nodeTimings = new Map<string, { [network: string]: FunctionStatsSummary }>();
+// Clients don't poll the runners directly, they poll their querysub server's index. Local state (not a plain variable), so the UI updates when a poll updates it.
+const polledIndexSchema = createLocalSchema("functionRunnerPolledIndex", {
+    index: t.atomic<FunctionRunnerIndex>(),
+});
+
+export function getFunctionRunnerIndex(): FunctionRunnerIndex | undefined {
+    void startFunctionRunnerTracking();
+    if (isClient()) {
+        // Until our first index poll finishes we use the index our edge node registered with (available immediately at startup, with no calls)
+        let bootedEdgeNode = (globalThis as any).BOOTED_EDGE_NODE as EdgeNodeConfig | undefined;
+        return Querysub.localRead(() => polledIndexSchema().index) || bootedEdgeNode?.functionRunnerIndex;
+    }
+    return getFunctionRunnerIndexServer();
+}
+export function getFunctionRunnerIndexServer(): FunctionRunnerIndex {
+    void startFunctionRunnerTracking();
+    let nodes = Array.from(nodeInfos.values());
+    sort(nodes, x => x.nodeId);
+    let callStats: FunctionStatsSummary = { count: 0, totalTime: 0, minTime: 0, maxTime: 0 };
+    let networkStatsObj: FunctionRunnerIndex["networkStats"] = {};
+    for (let timings of nodeTimings.values()) {
+        for (let [network, summary] of Object.entries(timings)) {
+            let networkSummary = networkStatsObj[network];
+            if (!networkSummary) {
+                networkSummary = { count: 0, totalTime: 0, minTime: 0, maxTime: 0 };
+                networkStatsObj[network] = networkSummary;
+            }
+            mergeSummary(networkSummary, summary);
+            mergeSummary(callStats, summary);
+        }
+    }
+    return {
+        nodes,
+        callStats,
+        networkStats: networkStatsObj,
+    };
+
+    function mergeSummary(into: FunctionStatsSummary, from: FunctionStatsSummary) {
+        if (from.count === 0) return;
+        if (into.count === 0) {
+            into.minTime = from.minTime;
+        }
+        into.count += from.count;
+        into.totalTime += from.totalTime;
+        into.minTime = Math.min(into.minTime, from.minTime);
+        into.maxTime = Math.max(into.maxTime, from.maxTime);
+    }
+}
+
+setImmediate(() => {
+    void import("./QuerysubController");
+});
+// Awaits getting the initial function runner information, so it is populated before anything makes calls (and so servers can include the initial index in their edge node config).
+export const startFunctionRunnerTracking = lazy(async () => {
+    if (isClient()) {
+        await runInfinitePollCallAtStart(INDEX_POLL_INTERVAL, async function pollFunctionRunnerIndex() {
+            const { QuerysubController, querysubNodeId } = await import("./QuerysubController");
+            let nodeId = await querysubNodeId();
+            if (!nodeId) return;
+            let index = await QuerysubController.nodes[nodeId].getFunctionRunnerIndex();
+            Querysub.localCommit(() => {
+                polledIndexSchema().index = index;
+            });
+        });
+        return;
+    }
+    console.log(`Starting function runner tracking (polling every ${formatTime(POLL_INTERVAL)})`);
+    await runInfinitePollCallAtStart(POLL_INTERVAL, pollFunctionRunners);
+    finishedStartup = true;
+});
+
+let finishedStartup = false;
+async function pollFunctionRunners() {
+    let now = Date.now();
+    let nodeIds = await getAllNodeIds();
+    await spreadCallsOverTime(nodeIds, finishedStartup ? POLL_INTERVAL : 0, async nodeId => {
+        let start = Date.now();
+        // Only function runners expose this, so a failed call just means the node isn't a runner
+        let runnerInfo = await timeoutToUndefinedSilent(POLL_INTERVAL, FunctionRunnerInfoController.nodes[nodeId].getRunnerInfo({
+            writeNodeId: getOwnNodeId(),
+        }));
+        if (!runnerInfo) return;
+        let latency = Date.now() - start;
+        if (runnerInfo.shards.length === 0) return;
+
+        let networks = new Set<string>();
+        for (let shard of runnerInfo.shards) {
+            for (let network of shard.networks) {
+                networks.add(network);
+            }
+        }
+
+        let prev = nodeInfos.get(nodeId);
+        let sampleCount = Math.min(prev?.latencySampleCount || 0, LATENCY_SAMPLE_LIMIT);
+        nodeInfos.set(nodeId, {
+            nodeId,
+            entryPoint: runnerInfo.entryPoint,
+            startupTime: runnerInfo.startupTime,
+            lastSeen: now,
+            isPublic: runnerInfo.shards.some(x => x.isPublic),
+            networks: Array.from(networks),
+            shards: runnerInfo.shards,
+            averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
+            latencySampleCount: sampleCount + 1,
+            scheduledShutdownTime: runnerInfo.scheduledShutdownTime,
+        });
+        nodeTimings.set(nodeId, runnerInfo.timings);
+        watchRunnerDisconnect(nodeId);
+    });
+
+    for (let [nodeId, info] of Array.from(nodeInfos)) {
+        if (info.lastSeen < now - NODE_EXPIRY_TIME) {
+            nodeInfos.delete(nodeId);
+            nodeTimings.delete(nodeId);
+        }
+    }
+
+    if (nodeInfos.size === 0) {
+        console.warn(`No function runner networks discovered at ${new Date().toLocaleString()}. Function calls cannot be given a network until one is found (we poll again in ${formatTime(POLL_INTERVAL)}).`);
+    }
+}
+
+// Remove disconnected runners immediately, otherwise we would keep recommending their networks for up to NODE_EXPIRY_TIME after every runner on them is gone. If the runner comes back, the poll re-adds it (and re-registers this watch).
+let disconnectWatchedNodes = new Set<string>();
+function watchRunnerDisconnect(nodeId: string) {
+    if (disconnectWatchedNodes.has(nodeId)) return;
+    disconnectWatchedNodes.add(nodeId);
+    SocketFunction.onNextDisconnect(nodeId, () => {
+        disconnectWatchedNodes.delete(nodeId);
+        if (nodeInfos.delete(nodeId)) {
+            nodeTimings.delete(nodeId);
+            console.log(`Function runner ${nodeId} disconnected, removing it from the runner index immediately`);
+        }
+    }, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
+}
+
+/** Whether any known runner on this network is public. Unknown networks count as non-public. */
+export function isNetworkPublic(network: string): boolean {
+    for (let info of nodeInfos.values()) {
+        if (info.networks.includes(network) && info.isPublic) return true;
+    }
+    return false;
+}
+
+/** Everything the selection algorithm knows about each network, so the UI can show why a network was (or wasn't) selected. */
+export function getNetworkSelectionInfos(): NetworkSelectionInfo[] {
+    let index = getFunctionRunnerIndex();
+    if (!index) return [];
+    let now = Date.now();
+
+    let perNetwork = new Map<string, FunctionRunnerNodeInfo[]>();
+    for (let node of index.nodes) {
+        for (let network of node.networks) {
+            let list = perNetwork.get(network);
+            if (!list) {
+                list = [];
+                perNetwork.set(network, list);
+            }
+            list.push(node);
+        }
+    }
+
+    let infos: NetworkSelectionInfo[] = [];
+    for (let [network, nodes] of perNetwork) {
+        let averageLatency = nodes.reduce((sum, x) => sum + x.averageLatency, 0) / nodes.length;
+        let stats = index.networkStats[network];
+        let averageCallTime = 0;
+        if (stats && stats.totalTime >= MIN_TOTAL_CALL_TIME_FOR_WEIGHTING && stats.count > 0) {
+            averageCallTime = stats.totalTime / stats.count;
+        }
+        let dying = nodes.every(x => x.scheduledShutdownTime !== undefined && x.scheduledShutdownTime - now < NETWORK_DYING_WINDOW);
+        infos.push({
+            network,
+            nodeCount: nodes.length,
+            isPublic: nodes.some(x => x.isPublic),
+            upLongEnough: nodes.some(x => now - x.startupTime >= MIN_UP_TIME),
+            dying,
+            averageLatency,
+            averageCallTime,
+            score: averageLatency * (averageCallTime + CALL_TIME_WEIGHT_BASE),
+        });
+    }
+    sort(infos, x => x.score);
+    return infos;
+}
+
+export function getAutoSelectedNetwork(): string | undefined {
+    let infos = getNetworkSelectionInfos();
+    // Networks where every runner is non-public must never be selected by default. Networks that will have no runners soon are forcefully switched off of.
+    let candidates = infos.filter(x => !x.dying);
+    if (candidates.length === 0) {
+        candidates = infos;
+    }
+    if (isPublic()) {
+        candidates = candidates.filter(x => x.isPublic);
+    }
+    let upLongEnough = candidates.filter(x => x.upLongEnough);
+    if (upLongEnough.length > 0) {
+        candidates = upLongEnough;
+    }
+    return candidates[0]?.network;
+}
+
+/** Safe to call from any context. Returns the network new function calls should be put on. Throws if no network is available (calls without a network can never run), unless noThrow is set. */
+export function getSelectedNetwork(): string;
+export function getSelectedNetwork(config: { noThrow: boolean }): string | undefined;
+export function getSelectedNetwork(config?: { noThrow?: boolean }): string | undefined {
+    let network = forcedNetworkURL.value || getAutoSelectedNetwork();
+    if (!network && !config?.noThrow) {
+        throw new Error(`No function runner discovered during discovery, so this call cannot be given a network and would never run (we keep polling for function runners, every ${formatTime(POLL_INTERVAL)} when tracking, every ${formatTime(INDEX_POLL_INTERVAL)} for the client index)`);
+    }
+    return network;
+}

package/src/4-querysub/Querysub.ts

@@ -290,7 +290,7 @@ export class Querysub {
      *          isn't presently necessary in most serverside scripts.
      */
     @measureFnc
-    public static async optionalStartupWait(addTime?: (name: string) => void) {
+    public static async startupWait(addTime?: (name: string) => void) {
         // Used by authorityLookup, pulled out here so timing is more clear.
         await onNodeDiscoveryReady();
         addTime?.("onNodeDiscoveryReady");
@@ -300,6 +300,10 @@ export class Querysub {
             await waitForFirstTimeSync();
         }, "waitForFirstTimeSync");
         addTime?.("waitForFirstTimeSync");
+        // Populate the function runner information (networks, timings), so calls can select a network immediately.
+        const { startFunctionRunnerTracking } = await import("./FunctionRunnerTracking");
+        await startFunctionRunnerTracking();
+        addTime?.("pollFunctionRunners");
     }
 
     public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
@@ -860,6 +864,9 @@ export class Querysub {
                 host: ipDomain + ":" + port,
                 entryPaths,
             });
+            // Watch the FunctionRunners in the cluster (networks, latency, call times), building the in-memory index clients pull (initially via the edge node config, then by polling QuerysubController) to select a network for their calls.
+            const { startFunctionRunnerTracking } = await import("./FunctionRunnerTracking");
+            void startFunctionRunnerTracking();
         }
     }
     private static async addSourceMapCheck(config: {
@@ -1080,7 +1087,7 @@ export class Querysub {
         await Promise.all([mountPromise, publishPromise]);
         addTime("mount & publish a records");
 
-        await Querysub.optionalStartupWait(addTime);
+        await Querysub.startupWait(addTime);
 
         console.log(magenta(`Started hosting service ${name}`) + ` | ${times.map(t => `${blue(t.name)}: ${green(formatTime(t.duration))}`).join(" | ")}`);
     }
@@ -1096,21 +1103,15 @@ export class Querysub {
          *  BOTH start and end must be provided, otherwise if only one is provided we will ignore it.
          */
         endFraction?: number;
-        /** Only listen to keys on this network. If not specified, only keys on the default network are returned. */
-        network?: string;
     }): (keyof T)[] {
 
-        let { startFraction, endFraction, network } = config || {};
+        let { startFraction, endFraction } = config || {};
         if ((startFraction === undefined) !== (endFraction === undefined)) {
             throw new Error(`startFraction and endFraction must both be provided, or both be undefined. If you want to get all keys, don't provide startFraction and endFraction.`);
         }
         if (!isNode() && startFraction !== undefined && endFraction !== undefined) {
             console.warn(`keys() with a range restriction is not supported clientside. It's too complicated for the proxy to handle it because the hashing depends on the authority server. You can synchronize all the keys client side, but you can't synchronize a restriction of the keys.`);
         }
-        if (network && startFraction === undefined) {
-            startFraction = 0;
-            endFraction = 1;
-        }
         if (startFraction === undefined || endFraction === undefined) {
             return Object.keys(obj);
         }
@@ -1118,15 +1119,10 @@ export class Querysub {
         if (!path) {
             return Object.keys(obj);
         }
-        let packedPath = encodeParentFilter({ path, startFraction, endFraction, network });
+        let packedPath = encodeParentFilter({ path, startFraction, endFraction });
         return proxyWatcher.getKeys(packedPath);
     }
 
-    /** Creates a key that lives on the given network. Reads and writes using the returned key route to authorities on that network (instead of the default network), and it will only be enumerated by Querysub.keys calls that ask for that network. */
-    public static createNetworkKey(config: { key: string; network: string }): string {
-        return createNetworkKey({ originalKey: config.key, network: config.network });
-    }
-
     // TODO: Maybe expose checkPermissions(getValue: () => unknown)?
     //  - It would be easy, if we every need to explicitly check if we have permissions. Although, it seems
     //      like just relying on the automatic checking is better?
@@ -1377,6 +1373,5 @@ import { onAllPredictionsFinished } from "../-0-hooks/hooks";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
 import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
 import { encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
-import { createNetworkKey } from "../0-path-value-core/PathRouterRouteOverride";
 import { AliveChecker, registerAliveChecker } from "../2-proxy/garbageCollection";
 import { QuerysubController, anyPredictionsPending, flushDelayedFunctions, onCallPredict, waitUntilAllPredictionsFinish } from "./QuerysubController";

package/src/4-querysub/QuerysubController.ts

@@ -34,9 +34,10 @@ import * as prediction from "./querysubPrediction";
 setFlag(require, "preact", "allowclient", true);
 
 import yargs from "yargs";
-import { setRoutingOverrideKeyNetwork } from "../0-path-value-core/PathRouterRouteOverride";
+import { getFunctionRunnerIndexServer, isNetworkPublic } from "./FunctionRunnerTracking";
+import type { FunctionRunnerIndex } from "./FunctionRunnerTracking";
 import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
-import { getDomain, getPrimaryNetwork, isBootstrapOnly, DEFAULT_NETWORK } from "../config";
+import { getDomain, isBootstrapOnly } from "../config";
 import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
@@ -551,16 +552,12 @@ export class QuerysubControllerBase {
         let callerCreatorId = IdentityController_getPubKeyShort(caller);
         call.callerIP = IdentityController_getSecureIP(caller);
 
-        if (call.network && call.network !== DEFAULT_NETWORK && !await isManagementUser()) {
-            throw new Error(`Caller is not a management user, and so does not have permissions to set the network on calls. Call ${debugCallSpec(call)}, network was "${call.network}"`);
-        }
-        // Clients don't know their network, so we decide it for them, rewriting the callId so the call routes to (and is picked up by) the right network.
         if (!call.network) {
-            let network = getPrimaryNetwork();
-            if (network !== DEFAULT_NETWORK) {
-                call.network = network;
-                call.CallId = setRoutingOverrideKeyNetwork(call.CallId, network);
-            }
+            throw new Error(`Call has no network, so it would never run (no FunctionRunner would pick it up). Call ${debugCallSpec(call)}`);
+        }
+        // Networks where every function runner is non-public are only usable by super users (they were never meant to receive public traffic, ex, a dev network).
+        if (!isNetworkPublic(call.network) && !await isManagementUser()) {
+            throw new Error(`Caller is not a super user, and so cannot add calls on the non-public network "${call.network}". Call ${debugCallSpec(call)}`);
         }
 
         if (Querysub.SIMULATE_LAG) {
@@ -628,6 +625,10 @@ export class QuerysubControllerBase {
         //  except the server being down, in which case, the client will gracefully timeout when it doesn't receive the confirmation
     }
 
+    public async getFunctionRunnerIndex(): Promise<FunctionRunnerIndex> {
+        return getFunctionRunnerIndexServer();
+    }
+
     public async getModulePath(config: {
         functionSpec: {
             DomainName: string;
@@ -720,6 +721,7 @@ export const QuerysubController = SocketFunction.register(
         watch: { compress: true, },
         unwatch: { compress: true, },
         addCall: { compress: true, },
+        getFunctionRunnerIndex: { compress: true, },
         // NOTE: Most of these debug functions are pretty innocuous. A lot of it is actually already exposed, and other parts of it is fine. It's fine for the user to know what node a path is on. It's fine for them to know how many value paths there are.
         debugGetPathNodeIds: {},
         debugGetNodeSpecs: {},

package/src/4-querysub/permissions.ts

@@ -163,6 +163,8 @@ export class PermissionsCheck {
         if (rootKey === "Calls" || rootKey === "Results") {
             // Don't allow "" call to be read, as this would allow running Object.values() to read all calls.
             if (path === pathParts.emptyKeyPath) return { permissionsPath: path, allowed: false };
+            // Calls/Results are nested one deeper by network, so we also have to block the empty key at both the network level and the callId level (otherwise Object.values() on a network would read all of its calls).
+            if (getPathIndex(path, DEPTH_TO_DATA) === "" || getPathIndex(path, DEPTH_TO_DATA + 1) === "") return { permissionsPath: path, allowed: false };
             // NOTE: A lot of the time, anyone will technically be able to read calls, as we don't inspect the call.
             //  However... they would need to know the id, which will be very hard to guess.
             //  - Also inspecting the call adds a lot of call overhead, so we would prefer not to do that.
@@ -208,6 +210,7 @@ class PermissionsCheckSchema {
         ModuleId: "",
         runAtTime: epochTime,
         argsEncoded: "",
+        network: "",
     };
 
     /** Converts a specific path to a more general path. All paths which map to this general path

package/src/4-querysub/querysubPrediction.ts

@@ -74,10 +74,10 @@ const getDevFunctionSpecFromCall = cacheJSONArgsEqual(async (call: {
 
 
 export function getCallResultPath(call: CallSpec) {
-    return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
+    return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.network][call.CallId]);
 }
 export function getCallResult(call: CallSpec) {
-    return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId];
+    return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.network][call.CallId];
 }
 
 /** Force predictions to run in the trigger order, so they can resolve an be added before

package/src/config.ts

@@ -63,6 +63,7 @@ let networkFileNetworks = lazy((): string[] => {
     return fs.readFileSync(path, "utf8").split("\n").map(x => x.trim()).filter(x => x);
 });
 
+/** The networks a function runner listens on, from the command line (--network / --networkfile). */
 export function getNetworks(): string[] {
     let networks = yargObj.network;
     if (!networks) {
@@ -78,9 +79,6 @@ export function getNetworks(): string[] {
     if (result.length === 0) return [DEFAULT_NETWORK];
     return result;
 }
-export function getPrimaryNetwork(): string {
-    return getNetworks()[0];
-}
 
 type QuerysubConfig = {
     domain?: string;