querysub 0.2.0

package/src/4-querysub/QuerysubController.ts

@@ -0,0 +1,620 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { cache, lazy } from "socket-function/src/caching";
+import { appendToPathStr, getPathDepth, getPathFromStr, getPathIndex, getPathStr1, getPathStr2, getPathStr3, hack_getPackedPathSuffix, removePathLastPart } from "../path";
+import { CHILD_CHECK_PREFIX, FunctionMetadata } from "../3-path-functions/syncSchema";
+import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+import { atomic, atomicObjectWrite, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { CallSpec, DEPTH_TO_DATA, FunctionSpec, debugCallSpec, functionSchema } from "../3-path-functions/PathFunctionRunner";
+import { PathValueController } from "../0-path-value-core/PathValueController";
+import { epochTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, pathWatcher, Time, WatchConfig, compareTime, debugTime, getNextTime, MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
+import { IdentityController_getMachineId, IdentityController_getPubKeyShort, IdentityController_getSecureIP } from "../-c-identity/IdentityController";
+import { getControllerNodeId } from "../-g-core-values/NodeCapabilities";
+import { getModuleFromConfig, watchModuleHotreloads } from "../3-path-functions/pathFunctionLoader";
+import { errorToUndefined, ignoreErrors, logErrors } from "../errors";
+import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
+import { PermissionsCheck } from "./permissions";
+import { parseArgs, writeCall } from "../3-path-functions/PathFunctionHelpers";
+import { delay } from "socket-function/src/batching";
+import { isNode, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { registerResource } from "../diagnostics/trackResources";
+
+// Always whitelist preact, as most of the time we want it clientside
+import "preact";
+import { setFlag } from "socket-function/require/compileFlags";
+import { CallerContextBase } from "socket-function/SocketFunctionTypes";
+import { isTrustedByNode } from "../-d-trust/NetworkTrust2";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { Querysub, id } from "./Querysub";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import debugbreak from "debugbreak";
+import { isDefined } from "../misc";
+import { isClient, isServer } from "../config2";
+import { PromiseObj } from "../promise";
+import { LoggingClient } from "../0-path-value-core/LoggingClient";
+import * as prediction from "./querysubPrediction";
+import { getCallResultPath } from "./querysubPrediction";
+import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+setFlag(require, "preact", "allowclient", true);
+
+export { Querysub, id };
+
+
+// NOTE: Eventually this will be split by domain, because the underlying getAllNodeIds
+//  will be split by domain.
+// NOTE: We don't need to shard querysub nodes within a domain, because... they don't
+//  store data, and don't run functions, and only really check permissions (which should be
+//  very lightweight, hopefully...)
+//  - I guess we track our watches, but even that... should be fairly light, and... there
+//      is little benefit to sharding that?
+export const querysubNodeId = lazy(() => getControllerNodeId(QuerysubController));
+
+setImmediate(() => {
+    async function querysubWatchLatest(config: WatchConfig) {
+        let nodeId = await querysubNodeId();
+        if (!nodeId) throw new Error("No querysub node found");
+        await QuerysubController.nodes[nodeId].watch(config);
+    }
+    async function querysubUnwatchLatest(config: WatchConfig) {
+        let nodeId = await querysubNodeId();
+        if (!nodeId) throw new Error("No querysub node found");
+        await QuerysubController.nodes[nodeId].unwatch(config);
+    }
+
+    let baseRemoteWatchFunction = RemoteWatcher.REMOTE_WATCH_FUNCTION;
+    RemoteWatcher.REMOTE_WATCH_FUNCTION = async (config, authorityId) => {
+        let weTrusted = isServer() && await errorToUndefined(isTrustedByNode(authorityId));
+        if (!weTrusted) {
+            await querysubWatchLatest(config);
+        } else {
+            await baseRemoteWatchFunction(config, authorityId);
+        }
+    };
+
+    let baseRemoteUnwatchFunction = RemoteWatcher.REMOTE_UNWATCH_FUNCTION;
+    RemoteWatcher.REMOTE_UNWATCH_FUNCTION = async (config, authorityIdBase) => {
+        let weTrusted = isServer() && await isTrustedByNode(authorityIdBase);
+        if (!weTrusted) {
+            logErrors(querysubUnwatchLatest(config));
+        } else {
+            logErrors(baseRemoteUnwatchFunction(config, authorityIdBase));
+        }
+    };
+});
+
+
+
+function permissionsHash(caller: Readonly<CallerContextBase>) {
+    let machineId = IdentityController_getMachineId(caller);
+    let ip = IdentityController_getSecureIP(caller);
+    // NOTE: We also need the nodeId, as while the permissions are the same per machineId + ip, each
+    //  process needs to be sent it's own values!
+    return getPathStr3(machineId, ip, caller.nodeId);
+}
+interface PermissionsObj {
+    nodes: Set<string>;
+
+    newPaths: Set<string>;
+    newParentPaths: Set<string>;
+
+    // permissionsPath => Set<path>
+    permissionsPathGroups: Map<string, {
+        allowed: boolean;
+        outdated: boolean;
+        originalPaths: Set<string>;
+        // testPath => parentPath
+        originalParentPaths: Map<string, string>;
+    }>;
+    pathToPermissionsPath: Map<string, string>;
+
+
+    checkNow: () => void;
+    dispose: () => void;
+}
+let permissionsPerNode = new Map<string, PermissionsObj>();
+registerResource("querysub|permissionsPerNode", permissionsPerNode);
+
+const lastCallTime = cache((nodeId: string): { value: Time } => {
+    SocketFunction.onNextDisconnect(nodeId, () => {
+        setTimeout(() => {
+            if (SocketFunction.isNodeConnected(nodeId)) return;
+            lastCallTime.clear(nodeId);
+        }, 1000 * 60);
+    });
+    return { value: epochTime };
+});
+
+export const isTrustedByDomain = cache(async function isTrustedByDomain(domain: string): Promise<boolean> {
+    // NOTE: When getControllerNodeId accepts a domain we will pass them the domain we are using.
+
+    // If we are trusted by one, we should be trusted by call!
+    //  TODO: We should probably handle trust revocation? Or... not? Because that would only happen
+    //      if we take a server offline, in which case... it would be off anyways.
+    let pathValueNodeId = await getControllerNodeId(PathValueController);
+    if (!pathValueNodeId) {
+        return false;
+        //throw new Error(`No PathValueController found`);
+    }
+    return isTrustedByNode(pathValueNodeId);
+});
+
+// NOTE: The last promise (added) ends up being the call we wait on, so that predictions
+//  can extend the wait time as they do more work.
+let pendingPredictedCalls = new Map<string, {
+    obj: PromiseObj;
+    seqNum: number;
+}>();
+export function callWaitOn(callId: string, promise: Promise<unknown>) {
+    let waitObj = pendingPredictedCalls.get(callId);
+    if (!waitObj) return;
+    let seqNum = ++waitObj.seqNum;
+    void promise.finally(() => {
+        if (waitObj?.seqNum === seqNum) {
+            waitObj.obj.resolve();
+        }
+    });
+}
+let baseWriteCall = writeCall.value;
+writeCall.value = async (callSpec: CallSpec, metadata: FunctionMetadata) => {
+    pendingPredictedCalls.set(callSpec.CallId, {
+        obj: new PromiseObj(),
+        seqNum: 0,
+    });
+    let promise = (async () => {
+        // IMPORTANT! While it may seem like we can use addCall even if we are trusted, so we can
+        //  have function calling prediction, we can't. If did then we would depend on these
+        //  predictions with our writes, which could end up on other nodes, which would then
+        //  be rejected (as our predictions use a Time.version that is only on our
+        //  node). The in unavoidable, as it is too difficult to have predictions, but also not have
+        //  predictions, on the same node.
+        //  - AND, we can't just use the same Time.version as the FunctionRunner will use, as our
+        //      ValuePaths might differ, which would sometimes work, BUT, other times our ReadLocks
+        //      would be valid, BUT, the actual read value would be different (as we might compute
+        //      a different value than FunctionRunner). So... we would lose locking, at which point,
+        //      why even use FunctionRunner in the first place! Just write the writes directly!
+        // TODO: Add the ability to use direct writes BUT if we have a rejection, commit it as a regular function call
+        //  - This will be a lot more efficient, although, if the committed dies and the function is rejected, the
+        //      value will be lost, which isn't great. However... trusted nodes shouldn't be dying so easily!
+        let isTrusted = isServer() && await isTrustedByDomain(callSpec.DomainName);
+        if (isTrusted) {
+            // NOTE: Direct writes should probably be used for calls from trusted nodes, which will
+            //  be a lot faster.
+            await baseWriteCall(callSpec, metadata);
+        } else {
+            await addCall(callSpec, metadata);
+        }
+    })();
+    callWaitOn(callSpec.CallId, promise);
+    await promise;
+};
+
+export async function onCallPredict(call: CallSpec | undefined) {
+    if (!call) return;
+    await pendingPredictedCalls.get(call.CallId)?.obj.promise;
+}
+export async function waitUntilAllPredictionsFinish() {
+    try {
+        await Promise.allSettled(Array.from(pendingPredictedCalls.values()).map(obj => obj.obj.promise));
+    } catch {
+    }
+}
+
+
+export async function flushDelayedFunctions() {
+    await prediction.flushDelayedFunctions();
+}
+
+export async function addCall(call: CallSpec, metadata: FunctionMetadata) {
+    await prediction.addCall(call, metadata);
+}
+
+export async function baseAddCall(call: CallSpec, nodeId: string, cancel: () => void, type: string) {
+    if (Querysub.DEBUG_CALLS) {
+        console.log(`[Querysub] ${magenta(type)}    ${green(debugCallSpec(call))}    @${debugTime(call.runAtTime)}`);
+    }
+    let errorWatcher = Querysub.createWatcher(() => {
+        let callResult = atomic(prediction.getCallResult(call));
+        if (callResult && "error" in callResult && Querysub.isAllSynced()) {
+            console.error(callResult.error.split("\n")[0] + `\n    (${call.DomainName}.${call.FunctionId})\n    ${callResult.error}`);
+            // Stop watching when we get the first error, otherwise other changes will trigger duplicate errors
+            //  (such as when the prediction is removed).
+            errorWatcher.dispose();
+        }
+    });
+    setTimeout(() => {
+        errorWatcher.dispose();
+    }, MAX_CHANGE_AGE);
+    try {
+        await QuerysubController.nodes[nodeId].addCall(call);
+    } catch (e) {
+        // Cancellation is IMPORTANT! Otherwise the prediction might never be rejected,
+        //  and so the client will keep stale data!
+        cancel();
+        throw e;
+    }
+}
+
+class QuerysubControllerBase {
+    public async watch(config: WatchConfig) {
+        for (let path of config.paths) {
+            Querysub.assertDomainAllowed(path);
+        }
+        for (let path of config.parentPaths) {
+            Querysub.assertDomainAllowed(path);
+        }
+
+        // Both watch it locally, AND remotely, so we will receive all the changes,
+        //  AND forward them to the watcher
+        let caller = SocketFunction.getCaller();
+        let callerId = caller.nodeId;
+        let callerMachineId = IdentityController_getMachineId(caller);
+        let callerIP = IdentityController_getSecureIP(caller);
+        let pHash = permissionsHash(caller);
+
+        if (Querysub.SIMULATE_LAG) {
+            await delay(Querysub.SIMULATE_LAG);
+        }
+
+        let permissionsObjIn = permissionsPerNode.get(pHash);
+        if (!permissionsObjIn) {
+
+            permissionsObjIn = {
+                nodes: new Set(),
+
+                newPaths: new Set(),
+                newParentPaths: new Set(),
+
+                permissionsPathGroups: new Map(),
+                pathToPermissionsPath: new Map(),
+
+                checkNow: () => { },
+                dispose: () => { },
+            };
+            const permissionsObj = permissionsObjIn;
+
+
+            SocketFunction.onNextDisconnect(callerId, () => {
+                permissionsPerNode.delete(pHash);
+                permissionsObj?.dispose();
+            });
+            // TODO: Once we have fast and easy incremental watches, use that to make this faster
+            let watcher = proxyWatcher.createWatcher({
+                watchFunction: this.createPermissionsCheck({ permissionsObj, callerMachineId, callerIP, callerId }),
+            });
+            watchModuleHotreloads(watcher);
+            permissionsObjIn.dispose = watcher.dispose;
+            permissionsObjIn.checkNow = watcher.explicitlyTrigger;
+            permissionsPerNode.set(pHash, permissionsObjIn);
+        }
+
+        for (let path of config.paths) {
+            permissionsObjIn.newPaths.add(path);
+        }
+        for (let path of config.parentPaths) {
+            permissionsObjIn.newParentPaths.add(path);
+        }
+
+        permissionsObjIn.checkNow();
+    }
+    private createPermissionsCheck(config: {
+        permissionsObj: PermissionsObj;
+        callerMachineId: string;
+        callerIP: string;
+        callerId: string;
+    }) {
+        const { permissionsObj, callerMachineId, callerIP, callerId } = config;
+        return function querysubCheckPermissions() {
+            if (!permissionsObj) throw new Error(`Internal error`);
+
+            let permissions = new PermissionsCheck({ callerMachineId, callerIP });
+
+            let extraPathsToCheck = new Set<string>();
+            let allowedPaths = new Set<string>();
+            let notAllowedPaths = new Set<string>();
+            let removedPaths = new Set<string>();
+
+            // HACK: Maintain a recording of parent paths separately, to make some code simpler.
+            let parentPathsLookup = new Map<string, string>();
+
+            const newPaths = permissionsObj.newPaths;
+            const newParentPaths = permissionsObj.newParentPaths;
+
+            function addPermissionsPath(path: string, parentPath: string | undefined) {
+                let checkResult = permissions.checkPermissions(path);
+                if (Querysub.anyUnsynced()) return;
+                let permissionsPath = checkResult.permissionsPath;
+                let group = permissionsObj.permissionsPathGroups.get(permissionsPath);
+                if (!group) {
+                    group = { allowed: checkResult.allowed, originalPaths: new Set(), originalParentPaths: new Map(), outdated: true };
+                    permissionsObj.permissionsPathGroups.set(permissionsPath, group);
+                }
+                if (parentPath !== undefined) {
+                    group.originalParentPaths.set(path, parentPath);
+                } else {
+                    group.originalPaths.add(path);
+                }
+
+                permissionsObj.pathToPermissionsPath.set(path, permissionsPath);
+            }
+
+            for (let path of newPaths) {
+                extraPathsToCheck.add(path);
+                addPermissionsPath(path, undefined);
+            }
+            for (let parentPath of newParentPaths) {
+                // NOTE: checkPermissions handles "" keys specially, but we still need to
+                //  turn them into a child key watch!
+                let testPath = appendToPathStr(parentPath, "");
+                parentPathsLookup.set(testPath, parentPath);
+                extraPathsToCheck.add(testPath);
+                addPermissionsPath(testPath, parentPath);
+            }
+
+            for (let [permissionsPath, obj] of permissionsObj.permissionsPathGroups) {
+                // NOTE: Even if it is already unsynced, the allowed flag might be wrong, so we have to check
+                //  it irregardless.
+                let allowed = permissions.checkPermissions(permissionsPath).allowed;
+                if (allowed !== obj.allowed && !Querysub.anyUnsynced()) {
+                    // NOTE: We don't reset outdated even if we return, that way if we run
+                    //  again we will check any previously outdated paths as well (otherwise we miss them).
+                    obj.outdated = true;
+                    obj.allowed = allowed;
+                }
+            }
+
+            // Don't commit anything if we have unsynced accesses after rechecking our permission paths
+            if (Querysub.anyUnsynced()) {
+                return;
+            }
+
+            // ONLY consume the paths if we are ready to apply them
+            permissionsObj.newPaths = new Set();
+            permissionsObj.newParentPaths = new Set();
+
+            let outdatedCount = 0;
+            for (let [permissionsPath, obj] of permissionsObj.permissionsPathGroups) {
+                if (!obj.outdated) continue;
+                // if (permissionsPath.startsWith(".,querysubtest._com.,PathFunctionRunner.,Page.,Data.,world.,grid")) {
+                //     console.log(magenta(`Changed permission path ${permissionsPath}, ${obj.allowed ? "allowed" : "not allowed"}`), obj.originalPaths);
+                // }
+                outdatedCount++;
+                obj.outdated = false;
+                for (let path of obj.originalPaths) {
+                    if (obj.allowed) allowedPaths.add(path);
+                    else {
+                        notAllowedPaths.add(path);
+                        removedPaths.add(path);
+                    }
+                }
+                for (let [path, parentPath] of obj.originalParentPaths) {
+                    parentPathsLookup.set(path, parentPath);
+                    if (obj.allowed) allowedPaths.add(path);
+                    else {
+                        notAllowedPaths.add(path);
+                        removedPaths.add(path);
+                    }
+                }
+            }
+
+            for (let path of extraPathsToCheck) {
+                let allowed = permissions.checkPermissions(path).allowed;
+                if (allowed) allowedPaths.add(path);
+                else notAllowedPaths.add(path);
+            }
+
+
+            function splitParentPaths(paths: Set<string>): [string[], string[]] {
+                let parentPaths = Array.from(paths).map(path => parentPathsLookup.get(path)).filter(isDefined);
+                // IMPORTANT! DO NOT filter parent paths out of regular paths. If we did this, then if they
+                //  tried to sync any direct path that includes [""], they would never receive the value!
+                let justPaths = Array.from(paths);
+                return [parentPaths, justPaths];
+            }
+
+            if (notAllowedPaths.size > 0) {
+                if (PermissionsCheck.DEBUG && Querysub.isAllSynced()) {
+                    let hackNotRootPaths = Array.from(notAllowedPaths).filter(path => getPathDepth(path) > DEPTH_TO_DATA);
+                    if (hackNotRootPaths.length > 0) {
+                        for (let path of hackNotRootPaths) {
+                            let test = permissions.checkPermissions(path);
+                            permissions.checkPermissions(test.permissionsPath);
+                        }
+                        console.log(red(`Disallowing watches due to disallowed permissions.`));
+                        for (let path of hackNotRootPaths) console.log(`    ${blue(path)}`);
+                        logErrors(LoggingClient.nodes[callerId].onMessage({
+                            type: "warn",
+                            message: `Disallowing ${hackNotRootPaths.length} watches due to disallowed permissions.`,
+                            context: (
+                                hackNotRootPaths.map(path => `    ${path}`).join("\n")
+                            ),
+                        }));
+                    }
+                }
+                let [newParentPathsNotAllowed, newPathsNotAllowed] = splitParentPaths(notAllowedPaths);
+                // Send them undefined values at epochTime
+                //  NOTE: We COULD send at time Date.now(), which would allow clobbering any existing values. This might
+                //      make the UI look cleaner (instead of showing stale values, it shows nothing)?
+                let undefinedValues: PathValue[] = newPathsNotAllowed.map(path => ({ path, value: undefined, canGCValue: true, time: epochTime, locks: [], lockCount: 0, valid: true, event: false }));
+
+                diskLog(`Disallowing PathValue watches due to disallowed permissions`, { count: newPathsNotAllowed.length, callerId });
+
+                ignoreErrors(pathValueSerializer.serialize(undefinedValues, { compress: Querysub.COMPRESS_NETWORK }).then(buffers =>
+                    PathValueController.nodes[callerId].forwardWrites(
+                        buffers,
+                        Array.from(newParentPathsNotAllowed),
+                    )
+                ));
+            }
+
+            if (removedPaths.size > 0) {
+                let [removedParentPaths, removedJustPaths] = splitParentPaths(removedPaths);
+                pathWatcher.unwatchPath({ callback: callerId, paths: removedJustPaths, parentPaths: removedParentPaths });
+            }
+            if (allowedPaths.size > 0) {
+                let [newParentPathsAllowed, newPathsAllowed] = splitParentPaths(allowedPaths);
+                let watchConfig: WatchConfig = { paths: newPathsAllowed, parentPaths: newParentPathsAllowed };
+                pathWatcher.watchPath({ callback: callerId, ...watchConfig });
+                remoteWatcher.watchLatest(watchConfig);
+            }
+        };
+    }
+    public async unwatch(config: WatchConfig) {
+        let pHash = permissionsHash(SocketFunction.getCaller());
+        let callerId = SocketFunction.getCaller().nodeId;
+        let permissionsObjIn = permissionsPerNode.get(pHash);
+        if (permissionsObjIn) {
+            const permissionsObj = permissionsObjIn;
+            function delPermissionsPath(path: string, parentPath: boolean) {
+                let permissionsPath = permissionsObj.pathToPermissionsPath.get(path);
+                if (!permissionsPath) return;
+                let group = permissionsObj.permissionsPathGroups.get(permissionsPath);
+                if (!group) return;
+                permissionsObj.pathToPermissionsPath.delete(path);
+                group.originalPaths.delete(path);
+                if (parentPath) {
+                    group.originalParentPaths.delete(path);
+                }
+                if (group.originalPaths.size === 0 && group.originalParentPaths.size === 0) {
+                    permissionsObj.permissionsPathGroups.delete(permissionsPath);
+                }
+            }
+
+            for (let path of config.paths) {
+                delPermissionsPath(path, false);
+            }
+            for (let path of config.parentPaths) {
+                delPermissionsPath(appendToPathStr(path, ""), true);
+            }
+        }
+        pathWatcher.unwatchPath({ callback: callerId, ...config });
+    }
+    public async addCall(call: CallSpec) {
+        if (Querysub.DEBUG_CALLS) {
+            console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
+        }
+        Querysub.assertDomainAllowed(getPathStr1(call.DomainName));
+
+        let caller = SocketFunction.getCaller();
+        let callerMachineId = IdentityController_getMachineId(caller);
+        let callerCreatorId = IdentityController_getPubKeyShort(caller);
+        call.callerIP = IdentityController_getSecureIP(caller);
+
+        if (Querysub.SIMULATE_LAG) {
+            await delay(Querysub.SIMULATE_LAG * 2);
+        }
+
+        // 0) Ensure the call is valid
+        //  - Verifies callerMachineId
+        //  - Uses MAX_ACCEPTED_CHANGE_AGE to check the write time
+        //  - Rejects out of order times
+        //  - Verify the time creatorId
+        if (call.callerMachineId !== callerMachineId) {
+            throw new Error(`Caller machineId does not match call's callerMachineId, ${call.callerMachineId} !== ${callerMachineId}`);
+        }
+        // NOTE: We use a lower threshold than FunctionRunner to account for extra time for FunctionRunner to receive the call.
+        //      We want to reject slow calls from the client promptly, as at this point we know it is a client issue. If the call
+        //      passes here FunctionRunner will actually update runAtTime for old calls, running them anyways. We would prefer
+        //      if slow clients don't depend on this behavior, as it is only intended to smooth over backend server issues,
+        //      not to adjust slow (or really, disconnected and then reconnected) clients.
+        let now = Date.now();
+        let callTimethreshold = now - MAX_ACCEPTED_CHANGE_AGE * 0.5;
+        if (call.runAtTime.time < callTimethreshold) {
+            throw new Error(`Call is too old, and will be dropped, ${debugCallSpec(call)}, ${call.runAtTime.time} < ${callTimethreshold}`);
+        }
+
+        let futureThreshold = now + Querysub.MAX_FUTURE_CALL_TIME;
+        if (call.runAtTime.time > futureThreshold) {
+            let futureTime = call.runAtTime.time - futureThreshold;
+            throw new Error(`Call is too far in the future, ${debugCallSpec(call)}, ${call.runAtTime.time}. Future ${futureTime} > ${Querysub.MAX_FUTURE_CALL_TIME}`);
+        }
+
+        // NOTE: There isn't really any reason to force this, as they could easily have multiple nodes/machines connected
+        //  (if they were attempting to cheat). So while running out of order might be annoying, and we would like to prevent it...
+        //  this check doesn't really accomplish that in any meaningful way (and instead breaks some valid use cases).
+        // let lastTime = lastCallTime(caller.nodeId);
+        // if (compareTime(call.runAtTime, lastTime.value) <= 0) {
+        //     throw new Error(`Call is out of order, ${call.runAtTime.time} <= ${lastTime.value.time}`);
+        // }
+        // lastTime.value = call.runAtTime;
+
+        if (call.runAtTime.creatorId !== callerCreatorId) {
+            throw new Error(`Call creatorId does not match caller creatorId, ${call.runAtTime.creatorId} !== ${callerCreatorId}`);
+        }
+
+        // 1) Ensure the caller has at least permissions on the data for this module
+        // NOTE: We could use commitFunction in watch as well, but... I think watches will benefit
+        //  more from the batching, where as calls need batching less (and commitFunction is easier to use).
+        let hasPermissions = await proxyWatcher.commitFunction({
+            watchFunction: function checkPermissions() {
+                let dataPath = getProxyPath(() => (functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Data as any)[Querysub.CALL_PERMISSIONS_KEY]);
+                let check = new PermissionsCheck(call);
+                return check.checkPermissions(dataPath).allowed;
+            },
+        });
+
+        if (!hasPermissions) {
+            throw new Error(`Caller does not have permission to call ${call.DomainName}.${call.ModuleId}`);
+        }
+
+        // NOTE: We don't wait for our writes to actually commit, because... there isn't any reason they shouldn't
+        //  except the server being down, in which case, the client can figure that out anyways?
+        proxyWatcher.writeOnly({
+            canWrite: true,
+            eventWrite: true,
+            doNotStoreWritesAsPredictions: true,
+            watchFunction: function writeCall() {
+                functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Calls[call.CallId] = atomicObjectWrite(call);
+            },
+        });
+    }
+
+    public async getModulePath(config: {
+        functionSpec: FunctionSpec;
+    }): Promise<string> {
+        let spec = config.functionSpec;
+        Querysub.assertDomainAllowed(getPathStr1(spec.DomainName));
+        let moduleConfig = await proxyWatcher.commitFunction({
+            watchFunction: function getModuleConfig() {
+                return functionSchema()[spec.DomainName].PathFunctionRunner[spec.ModuleId].Sources[spec.FunctionId];
+            }
+        });
+        if (!moduleConfig) {
+            throw new Error(`Function ${spec.DomainName}.${spec.ModuleId}.${spec.FunctionId} does not exist (a deploy might be required).`);
+        }
+
+        let module = await getModuleFromConfig(moduleConfig);
+        return module.filename;
+    }
+
+
+    // NOTE: Maybe remove this? It's very useful for debugging though...
+    public async debugGetSingleReadNode(path: string) {
+        return pathValueAuthority2.getSingleReadNodeSync(path);
+    }
+
+    public async debugGetPathAuthorities(nodeId: string) {
+        return pathValueAuthority2.getAuthorityPaths(nodeId);
+    }
+}
+
+export const QuerysubController = SocketFunction.register(
+    "QuerysubController-6db5ef05-7563-4473-a440-2f64f03fe6ef",
+    new QuerysubControllerBase(),
+    () => ({
+        // TODO: Maybe allow compression to be enable (or disabled) at a per call / schema level?
+        watch: { compress: true, },
+        unwatch: { compress: true, },
+        addCall: { compress: true, },
+        debugGetSingleReadNode: {},
+        debugGetPathAuthorities: {},
+        getModulePath: {},
+    }),
+    () => ({
+
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+