querysub 0.403.0 → 0.405.0

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,766 +1,943 @@
-import { delay, runInfinitePoll } from "socket-function/src/batching";
-import { cache } from "socket-function/src/caching";
-import { blue, magenta, red, yellow } from "socket-function/src/formatting/logColors";
-import { sha256HashBuffer, timeInHour, timeInSecond } from "socket-function/src/misc";
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { getLowUint32, getShortNumber } from "../bits";
-import { registerDynamicResource } from "../diagnostics/trackResources";
-import { errorToUndefined, logErrors } from "../errors";
-import { getPathDepth, getPathFromStr, getPathIndex, trimPathStrToDepth } from "../path";
-import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
-import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { getProxyPath } from "../2-proxy/pathValueProxy";
-import { atomic, atomicObjectRead, atomicObjectWrite, doAtomicWrites, doProxyOptions, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { __getRoutingHash, authorityStorage, compareTime, debugTime, epochTime, getNextTime, isCoreQuiet, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
-import { getModuleFromSpec, watchModuleHotreloads } from "./pathFunctionLoader";
-import debugbreak from "debugbreak";
-import { parseArgs } from "./PathFunctionHelpers";
-import { FunctionMetadata, PERMISSIONS_FUNCTION_ID, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
-import { formatTime } from "socket-function/src/formatting/format";
-import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
-import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { getDomain, isLocal } from "../config";
-import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
-import { DeployProgress } from "../4-deploy/deployFunctions";
-import { logDisk } from "../diagnostics/logs/diskLogger";
-
-export const functionSchema = rawSchema<{
-    [domainName: string]: {
-        PathFunctionRunner: {
-            [ModuleId: string]: {
-                Data: unknown;
-                Sources: { [FunctionId: string]: FunctionSpec | undefined; };
-                Calls: { [CallId: string]: CallSpec | undefined; };
-                Results: { [CallId: string]: FunctionResult | undefined; };
-            };
-        };
-    };
-}>();
-
-export function commitCall(call: CallSpec) {
-    let domainName = call.DomainName;
-    let moduleId = call.ModuleId;
-    let callId = call.CallId;
-    proxyWatcher.writeOnly({
-        canWrite: true,
-        eventWrite: true,
-        doNotStoreWritesAsPredictions: true,
-        // NOTE: We write the call in the present, it is callSpec.runAtTime that is in the past. No one cares
-        //  when the call was added, only when it wants to run at.
-        watchFunction: function writeCall() {
-            functionSchema()[domainName].PathFunctionRunner[moduleId].Calls[callId] = atomicObjectWrite(call);
-        },
-    });
-}
-
-export const DEPTH_TO_DATA = 4;
-export const DOMAIN_INDEX = 0;
-export const MODULE_INDEX = 2;
-
-export function getSchemaPartsFromPath(path: string): { domainName: string; moduleId: string } | undefined {
-    let parts = getPathFromStr(path);
-    let domainName = parts[0];
-    let moduleId = parts[2];
-    if (!domainName || !moduleId) return undefined;
-    return { domainName, moduleId };
-}
-
-export type FunctionSpec = {
-    DomainName: string;
-    ModuleId: string;
-    FunctionId: string;
-    exportPathStr: string;
-    FilePath: string;
-    gitURL: string;
-    /** Ex, the hash */
-    gitRef: string;
-
-    maxLocksOverride?: number;
-};
-
-export interface CallSpec {
-    DomainName: string;
-    ModuleId: string;
-    CallId: string;
-    FunctionId: string;
-    // NOTE: This is no longer JSON. Use parseArgs to decode it
-    argsEncoded: string;
-    callerMachineId: string;
-    callerIP: string;
-    runAtTime: Time;
-
-    // Not just used for debugging, also used to add special proxy-related warnings. 
-    fromProxy?: string;
-
-    filterable?: Filterable;
-}
-export function debugCallSpec(spec: CallSpec): string {
-    return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
-}
-export type FunctionResult = ({
-
-} | {
-    error: string;
-}) & {
-    timeTaken: number;
-    evalTime: number;
-    runCount: number;
-};
-
-let currentCallSpec: { spec: CallSpec; fnc: FunctionSpec; } | undefined;
-export function getCurrentCall() {
-    if (!currentCallSpec) {
-        debugbreak(2);
-        debugger;
-        throw new Error("Not presently in a call, and so cannot get call");
-    }
-    return currentCallSpec.spec;
-}
-export function getCurrentCallAllowUndefined() {
-    return currentCallSpec?.spec;
-}
-export function getCurrentCallObj() {
-    return currentCallSpec;
-}
-/** NOTE: We require the FunctionSpec so we know what has is loaded, to identify the
- *      correct schema. This should be available, but if it isn't... we MIGHT be able
- *      to work around it, if we can be assured the schema won't be used?
- *      - This is so pre-loading of functions (which might happen without a deploy, if the deploy script
- *          is killed part of the way through), doesn't change schemas until the deploy actually
- *          updates the underlying functions.
-  */
-export function overrideCurrentCall<T>(config: { spec: CallSpec; fnc: FunctionSpec; }, code: () => T) {
-    let prev = currentCallSpec;
-    currentCallSpec = config;
-    try {
-        return code();
-    } finally {
-        currentCallSpec = prev;
-    }
-}
-
-function getDebugName(call: CallSpec, functionSpec: FunctionSpec, colored = false) {
-    let pathName = getPathFromStr(functionSpec.exportPathStr).slice(1).join(".");
-    let mainPart = `${call.DomainName}:${functionSpec.FilePath}|${pathName}|${debugTime(call.runAtTime)}`;
-    if (colored) {
-        return `${magenta(mainPart)}`;
-    } else {
-        return `${mainPart}`;
-    }
-}
-
-type PermissionsCheckType = {
-    new(callerMachineId: { callerMachineId: string; callerIP: string; }): {
-        checkPermissions(path: string): { permissionsPath: string; allowed: boolean; };
-    };
-    skipPermissionsChecks: <T>(code: () => T) => T;
-}
-
-export type DebugFunctionShardInfo = {
-    domainName: string;
-    shardRange: { startFraction: number, endFraction: number };
-    secondaryShardRange?: { startFraction: number, endFraction: number };
-};
-let debugFunctionRunnerShards: DebugFunctionShardInfo[] = [];
-export function debug_getFunctionRunnerShards() {
-    return debugFunctionRunnerShards;
-}
-setImmediate(() => {
-    set_debug_getFunctionRunnerShards(debug_getFunctionRunnerShards);
-});
-
-export class PathFunctionRunner {
-    public static RUN_START_COUNT = 0;
-    public static RUN_FINISH_COUNT = 0;
-    public static DEBUG_CALLS = false;
-    public static DEBUG_CALL_TRIGGERS = false;
-    public static DEBUG_WATCHES_THRESHOLD = 5;
-    public static MAX_WATCH_LOOPS = 1000;
-
-    constructor(private config: {
-        domainName: string;
-        shardRange: { startFraction: number, endFraction: number };
-        secondaryShardRange?: { startFraction: number, endFraction: number };
-        PermissionsChecker: PermissionsCheckType | undefined;
-        filterSelector?: FilterSelector;
-    }) {
-        SocketFunction.expose(FunctionPreloadController);
-        debugFunctionRunnerShards.push({
-            domainName: config.domainName,
-            shardRange: config.shardRange,
-            secondaryShardRange: config.secondaryShardRange,
-        });
-        logErrors(this.startWatching());
-    }
-
-    // Calls runCall on any calls that match our domain + shard range
-    private async startWatching() {
-        const { Querysub } = await import("../4-querysub/Querysub");
-        let { shardRange, secondaryShardRange } = this.config;
-
-        // We will use PathValueProxyWatcher to watch the paths
-        // Maintain only one outstanding runCall per callId
-        //  - Once it finishes, we can trigger another call, if we notice the result is not set.
-        //      - AS we should get write prediction for the result, we can rerun our proxy check
-        //          loop immediately.
-
-        let fullFraction = shardRange;
-        if (secondaryShardRange) {
-            fullFraction = {
-                startFraction: Math.min(shardRange.startFraction, secondaryShardRange.startFraction),
-                endFraction: Math.max(shardRange.endFraction, secondaryShardRange.endFraction),
-            };
-        }
-
-        let config = this.config;
-
-        let self = this;
-
-        let outstandingCalls = 0;
-
-        let watchModuleCalls = cache((moduleId: string) => {
-            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls);
-            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results);
-            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, fullFraction);
-            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, fullFraction);
-        });
-
-        let runningCalls = new Set<string>();
-
-        // Rarely we might need to queue a function multiple times, when we are late to receive rejections.
-        //  HOWEVER, after enough times, we should stop, as we will probably just infinitely queue it.
-        const MAX_QUEUE_COUNT = 100;
-        let queueLimitCounts = new Map<string, number>();
-        // Clear every hour, so we don't leave
-        runInfinitePoll(timeInHour, () => queueLimitCounts.clear());
-
-        const domainName = config.domainName;
-        const watcher = proxyWatcher.createWatcher({
-            trackTriggers: true,
-            skipPermissionsCheck: true,
-            hasSideEffects: true,
-            // Maybe allow running with unsynced reads? This might be good, because we don't need all reads
-            //  to detect calls. BUT, it might also be bad, because it might cause us to rerun in a cascading
-            //  fashion? (As in, be really slow).
-            //  - Shouldn't be needed, because writes don't take that long
-            //  - Could fix bugs where we never get a read, and therefore all function calls fails to run?
-            // allowUnsyncedReads: true
-            watchFunction: function findFunctionsToCall() {
-                // TODO: Maybe abstract the "preserve watchers" stuff,
-                //  so that it is automatically done at the start of our trigger.
-                const watcher = proxyWatcher.getTriggeredWatcher();
-
-                // Preserve previous watches explicitly (so we just do a diff, instead of accessing every
-                //  value every time).
-                proxyWatcher.reuseLastWatches();
-
-                let data = () => functionSchema()[domainName].PathFunctionRunner;
-
-                let moduleIds = Object.keys(data());
-                if (isLocal()) {
-                    moduleIds.push(...getAllDevelopmentModulesIds());
-                    moduleIds = Array.from(new Set(moduleIds));
-                }
-
-                for (let moduleId of moduleIds) {
-                    watchModuleCalls(moduleId);
-                }
-
-                let newCalls: { call: CallSpec; functionSpec: FunctionSpec }[] = [];
-
-                if (watcher.triggeredByChanges) {
-                    for (let path of watcher.triggeredByChanges.paths) {
-                        let callsOrResult = getPathIndex(path, 3);
-                        if (callsOrResult !== "Calls" && callsOrResult !== "Results") continue;
-                        if (getPathIndex(path, 1) !== "PathFunctionRunner") continue;
-                        if (getPathIndex(path, 0) !== domainName) continue;
-
-                        const callId = getPathIndex(path, 4);
-                        if (!callId) continue;
-                        let moduleId = getPathIndex(path, 2);
-                        if (!moduleId) continue;
-                        if (!moduleIds.includes(moduleId)) continue;
-
-                        let moduleData = data()[moduleId];
-
-                        let result = atomicObjectRead(moduleData.Results[callId]);
-                        if (result) continue;
-
-                        let callData = atomicObjectRead(moduleData.Calls[callId]);
-                        if (!callData) continue;
-
-                        // Ignore invalid data
-                        if (callData.DomainName !== domainName) continue;
-                        if (callData.ModuleId !== moduleId) continue;
-                        if (callData.CallId !== callId) continue;
-
-                        const functionId = callData.FunctionId;
-                        let functionSpec = atomicObjectRead(moduleData.Sources[functionId]);
-                        if (!functionSpec) {
-                            if (!isSynced(moduleData.Sources[callData.FunctionId])) {
-                                continue;
-                            }
-                            if (isLocal()) {
-                                functionSpec = getDevFunctionSpecFromCall(callData);
-                                if (!functionSpec) continue;
-                            } else {
-                                console.warn(yellow(`Cannot call ${callData.DomainName}.${callData.ModuleId}.Sources.${callData.FunctionId} because it is not deployed`));
-                                continue;
-                            }
-                        }
-
-                        // If we haven't synced the result, we can't know if it is ready or not
-                        //  (we'll run anyways once it is synced). Most other parts are immutable,
-                        //  but the result is by definition not, so... it is one of the values we check!
-                        if (!isSynced(moduleData.Results[callId])) continue;
-
-                        if (runningCalls.has(callId)) continue;
-                        runningCalls.add(callId);
-
-                        let limitCount = queueLimitCounts.get(callId) || 0;
-                        limitCount++;
-                        queueLimitCounts.set(callId, limitCount);
-                        // NOTE: Calls are event writes, so... they should just clean themselves up, after we ignore them for long enough.
-                        if (limitCount >= MAX_QUEUE_COUNT) {
-                            // Only error the first time, as we don't need need that many errors
-                            if (limitCount === MAX_QUEUE_COUNT) {
-                                console.error(`Tried to requeue a function to run too many times (${limitCount}) for ${getDebugName(callData, functionSpec, true)}. This is NOT due to cascading reads. It might be due to repeated rejections?`);
-                            }
-                            continue;
-                        }
-
-                        if (PathFunctionRunner.DEBUG_CALLS) {
-                            console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
-                            let resultsPath = getProxyPath(() => moduleData.Results[callId]);
-                            let history = authorityStorage.getValuePlusHistory(resultsPath);
-                            console.log(`    History: ${history.length}`);
-                            for (let { valid, time, canGCValue } of history) {
-                                console.log(`        ${valid ? "✅" : "❌"} ${debugTime(time)} ${canGCValue ? "value is undefined" : ""}`);
-                            }
-                        }
-                        newCalls.push({ call: callData, functionSpec });
-                    }
-                }
-
-                // Better run first calls first, or else we will just pointlessly cause rejections.
-                newCalls.sort((a, b) => compareTime(a.call.runAtTime, b.call.runAtTime));
-                for (let { call, functionSpec } of newCalls) {
-                    // We don't need wait for all pending synced accesses to finish.
-                    //  Worst case we have an old functionSpec, and then runCall has to loop again.
-                    //  Otherwise the call request is (or should be) atomic, so it this should just work...
-                    //  - Or if it is rejected, we will detect it has no result, and run it again.
-                    // NOTE: We run all calls in parallel, so we can load data in parallel. This is a bit dangerous, and might
-                    //      result in out of order execution, but... we commit at runTimes, so, this should eventually
-                    //      resolve itself due to rejections (which will change the call result, which SHOULD cause us to rerun again).
-                    //      - This is a bit dangerous, as we might overwhelm ourself. We should address this one we support
-                    //          FunctionRunner sharding.
-                    outstandingCalls++;
-                    let runCallPromise = self.runCall(call, functionSpec);
-                    void runCallPromise.finally(() => {
-                        outstandingCalls--;
-                        // IMPORTANT! We remove the call, NOT for GC, but! Because if the call writes are rejected (due to contention), then the Result will also be rejected, changing it back to undefined, so (as long as little enough time has passed), we will run the call again! Or rather, we will try to, as long as runningCalls doesn't have the value!
-                        runningCalls.delete(call.CallId);
-                    });
-                    logErrors(runCallPromise);
-                }
-            },
-        });
-        registerDynamicResource("paths|PathFunctionRunner.watches", () => watcher.lastWatches.paths.size);
-        registerDynamicResource("paths|PathFunctionRunner.outstandingCalls", () => outstandingCalls);
-
-        // If local, periodically check all modules
-        //  NOTE: `yarn deploy` commits the moduleIds to the database, but calls do not, as this would require either an extra
-        //      read, or an extra write. So this is somewhat a hack, but also fairly safe.
-        if (isLocal()) {
-            let prevModuleIds = new Set<string>();
-            runInfinitePoll(timeInSecond * 15, () => {
-                let moduleIds = getAllDevelopmentModulesIds();
-                let changed = false;
-                for (let moduleId of moduleIds) {
-                    if (!prevModuleIds.has(moduleId)) {
-                        changed = true;
-                        break;
-                    }
-                }
-                prevModuleIds = new Set(moduleIds);
-                if (changed) {
-                    watcher.explicitlyTrigger();
-                }
-            });
-        }
-    }
-
-
-    @measureFnc
-    private async getExportsFromSpec(spec: FunctionSpec): Promise<unknown> {
-        let module = await getModuleFromSpec(spec);
-        return module.exports;
-    }
-
-    private getSecondaryShardDelay(callPath: CallSpec): number {
-        let { shardRange, secondaryShardRange } = this.config;
-        if (!secondaryShardRange) return 0;
-        let fraction = __getRoutingHash(callPath.CallId);
-        // It isn't secondary if it is primary
-        if (shardRange.startFraction <= fraction && fraction < shardRange.endFraction) return 0;
-        if (!(secondaryShardRange.startFraction <= fraction && fraction < secondaryShardRange.endFraction)) return 0;
-        let fractionDistanceToPrimary = Math.min(
-            Math.abs(fraction - shardRange.startFraction),
-            Math.abs(fraction - shardRange.endFraction),
-        );
-        return Math.min(
-            // No more than 50% of accepted change, otherwise it might not be accepted
-            MAX_ACCEPTED_CHANGE_AGE * 0.5,
-            // Default values in case MAX_ACCEPTED_CHANGE_AGE is VERY high (minutes, or even hours)
-            30_000 + 10_000 * fractionDistanceToPrimary,
-            // Wait 5000 seconds, then start 
-            5000 + MAX_ACCEPTED_CHANGE_AGE * fractionDistanceToPrimary * 0.05 / (shardRange.endFraction - shardRange.startFraction),
-        );
-    };
-
-    private callsRemainingInTick = cache((wait: true) => {
-        // NOTE: This gives us 100K calls per second, which is more than we can run anyways.
-        //  We just need to wait a bit, otherwise the IO loop can never run, and so our finished calls
-        //  can never actually finish, until eventually they are too late to finish and get rejected!
-        let tickDone = delay(10);
-        void tickDone.finally(() => {
-            this.callsRemainingInTick.clear(true);
-        });
-        return { calls: 1000, tickDone };
-    });
-    private async letIOClear(): Promise<void> {
-        while (true) {
-            let callsObj = this.callsRemainingInTick(true);
-            callsObj.calls--;
-            if (callsObj.calls >= 0) return;
-            await callsObj.tickDone;
-        }
-    }
-
-    private async runCall(callPath: CallSpec, functionSpec: FunctionSpec): Promise<void> {
-        const { Querysub } = await import("../4-querysub/Querysub");
-
-        const PermissionsChecker = this.config.PermissionsChecker;
-        let skipPermissions = <T>(code: () => T) => code();
-        if (PermissionsChecker) {
-            skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
-        }
-
-        if (!doesMatch(callPath.filterable, this.config.filterSelector)) {
-            return;
-        }
-
-        await this.letIOClear();
-
-        let secondaryDelay = this.getSecondaryShardDelay(callPath);
-        if (secondaryDelay) {
-            async function getIsFinished() {
-                return await proxyWatcher.commitFunction({
-                    watchFunction() {
-                        let syncedModule = skipPermissions(() =>
-                            functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId]
-                        );
-                        return !!atomicObjectRead(syncedModule.Results[callPath.CallId]);
-                    },
-                });
-            }
-            // Call once, so we are already watching the 
-            await delay(secondaryDelay);
-            let isFinished = await getIsFinished();
-            if (isFinished) return;
-            if (PathFunctionRunner.DEBUG_CALLS) {
-                let fraction = __getRoutingHash(callPath.CallId);
-                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${getDebugName(callPath, functionSpec, true)}`);
-            }
-        }
-
-        PathFunctionRunner.RUN_START_COUNT++;
-        if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`STARTING ${getDebugName(callPath, functionSpec, true)}`);
-        }
-        let startTime = Date.now();
-        let runCount = 0;
-
-        let finalWrites: PathValue[] | undefined;
-
-        let evalTime = 0;
-        let retries = 0;
-
-        let nooped = false;
-
-        while (true) {
-            let isFunctionSpecOutdated = false;
-            try {
-
-                let moduleExports = await this.getExportsFromSpec(functionSpec);
-
-                // ALWAYS get the permissions as well. This isn't perfect, as we could very well access data in other files, but... it helps
-                //  reduce extra iterations inside of commitFunction...
-                await getModuleFromSpec({ ...functionSpec, exportPathStr: getExportPath(PERMISSIONS_FUNCTION_ID), FunctionId: PERMISSIONS_FUNCTION_ID, });
-
-                let exportPath = getPathFromStr(functionSpec.exportPathStr);
-                let exportObj = moduleExports as any;
-                for (let path of exportPath) {
-                    exportObj = exportObj[path];
-                }
-                let baseFunction = exportObj as Function;
-
-                if (typeof baseFunction !== "function") {
-                    throw new Error(`Export at ${JSON.stringify(exportPath.join("."))} was not a function (it was ${typeof baseFunction})`);
-                }
-
-                let devFunctionMetadata: FunctionMetadata | undefined;
-
-                let devModule = getDevelopmentModule(functionSpec.ModuleId);
-                if (devModule) {
-                    let schemaObj = getSchemaObject(devModule);
-                    devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
-                }
-
-                await proxyWatcher.commitFunction({
-                    canWrite: true,
-                    debugName: getDebugName(callPath, functionSpec),
-                    runAtTime: callPath.runAtTime,
-                    getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callPath)),
-                    nestedCalls: "inline",
-                    temporary: true,
-                    maxLocksOverride: devFunctionMetadata ? devFunctionMetadata.maxLocksOverride : functionSpec.maxLocksOverride,
-                    watchFunction: function runCallWatcher() {
-                        runCount++;
-                        if (PathFunctionRunner.DEBUG_CALLS) {
-                            console.log(`Evaluating (try count ${runCount}) ${getDebugName(callPath, functionSpec, true)}`);
-                        }
-                        if (PathFunctionRunner.DEBUG_CALL_TRIGGERS && runCount > PathFunctionRunner.DEBUG_WATCHES_THRESHOLD) {
-                            // NOTE: If this happens a few times during initial loading it is fine. If it happens a lot... and with values MUCH
-                            //  greater than 2... it is likely a problem, and your functions might be doing cascading loading, which can take
-                            //  a very long time (ex, `for(let i = 0; i < 1000; i++) if (!arr[i]) return`)
-                            console.log(`> 2 call evaluations (try count ${runCount}) ${getDebugName(callPath, functionSpec, true)}`);
-                            console.log(yellow(`    Paths changed`));
-                            for (let path of proxyWatcher.getTriggeredWatcher().triggeredByChanges?.paths || []) {
-                                console.log(yellow(`        ${path}`));
-                            }
-                        }
-                        if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
-                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`);
-                        }
-
-                        // We NEED to depend on the function spec, so spec updates can be atomic (which is needed to deploy an update
-                        //  to many nodes at once, without having any period where multiple versions both commit values).
-                        let syncedModule = skipPermissions(() =>
-                            functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId]
-                        );
-
-                        // Make sure we are running the latest function (by checking it here, we also lock the values,
-                        //  so if it is updated we atomically rerun / reject all calls which were racing with us).
-                        if (!isLocal()) {
-                            let syncedSpec = skipPermissions(() =>
-                                atomicObjectRead(syncedModule.Sources[callPath.FunctionId])
-                            );
-                            if (!syncedSpec) {
-                                throw new Error(`Function spec not found for ${getDebugName(callPath, functionSpec, true)}`);
-                            }
-
-                            //  (We also need to depend on the RIGHT function spec).
-                            if (
-                                syncedSpec && (
-                                    syncedSpec.DomainName !== functionSpec.DomainName
-                                    || syncedSpec.ModuleId !== functionSpec.ModuleId
-                                    || syncedSpec.FilePath !== functionSpec.FilePath
-                                    || syncedSpec.FunctionId !== functionSpec.FunctionId
-                                    || syncedSpec.exportPathStr !== functionSpec.exportPathStr
-                                    || syncedSpec.FilePath !== functionSpec.FilePath
-                                    || syncedSpec.gitRef !== functionSpec.gitRef
-                                    || syncedSpec.gitURL !== functionSpec.gitURL
-                                    || syncedSpec.maxLocksOverride !== functionSpec.maxLocksOverride
-                                )
-                            ) {
-                                isFunctionSpecOutdated = true;
-                                functionSpec = {
-                                    DomainName: syncedSpec.DomainName,
-                                    ModuleId: syncedSpec.ModuleId,
-                                    FilePath: syncedSpec.FilePath,
-                                    FunctionId: syncedSpec.FunctionId,
-                                    exportPathStr: syncedSpec.exportPathStr,
-                                    gitRef: syncedSpec.gitRef,
-                                    gitURL: syncedSpec.gitURL,
-                                };
-                                return;
-                            }
-                        }
-
-                        let evalTimeStart = Date.now();
-                        try {
-                            let args = parseArgs(callPath);
-                            overrideCurrentCall({ spec: callPath, fnc: functionSpec, }, () => {
-                                baseFunction(...args);
-                            });
-                        } finally {
-                            evalTime += Date.now() - evalTimeStart;
-                        }
-
-                        // NOTE: The results are only temporarily stored, but serve as a way to let clients
-                        //  know figure out the dependencies for the call, which makes function call prediction
-                        //  possible.
-                        skipPermissions(() => {
-                            let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callPath.CallId]));
-                            if (result) {
-                                nooped = true;
-                                // It is important to NOOP if it has already been written. Otherwise there could be multiple
-                                //  runs of the same function, which could easily vary (if they use Math.random(), etc).
-                                //  WHICH, as the write time is predetermined, would cause ValuePaths with equal times, but
-                                //  different values (which breaks a lot of assumptions much of the code depends on! such
-                                //  as locks, etc).
-                                if (!secondaryDelay && Querysub.isAllSynced()) {
-                                    console.warn(`Skipping function write, as it has already been written to by another FunctionRunner.`);
-                                }
-                                return;
-                            }
-                            nooped = false;
-                            syncedModule.Results[callPath.CallId] = atomicObjectWrite({
-                                runCount: runCount,
-                                timeTaken: Date.now() - evalTimeStart,
-                                evalTime,
-                            });
-                            proxyWatcher.setEventPath(() => syncedModule.Results[callPath.CallId]);
-                        });
-
-                    },
-                }, {
-                    onWritesCommitted(writes) {
-                        finalWrites = writes;
-                    },
-                });
-            } catch (e: any) {
-                if (isFunctionSpecOutdated) continue;
-
-                // NOTE: We are changing our paradigm to reject early, rather than retry. This is more stable. You can't really commit
-                //  important values and then just leave, unless those values have no locks. This helps prevent values from being
-                //  in different states on different machines, by just rejecting the call.
-                // if (e.message.includes("MAX_CHANGE_AGE_EXCEEDED") && retries < 10 && evalTime < MAX_ACCEPTED_CHANGE_AGE) {
-                //     // NOTE: This should never really happen. It is only really if our PathValueServer goes down for a minute or so,
-                //     //  so we don't just drop all of our function calls.
-                //     //  - This is why we log every time, because no user actions should be able to result in this happening frequently.
-                //     console.error(red(`RETRYING due to MAX_CHANGE_AGE_EXCEEDED. Hopefully the server will be more responsive this time (retry ${retries}): ${getDebugName(callPath, functionSpec, true)}`));
-                //     // NOTE: Updating the time is fine, and won't break prediction rejections, as we reject based on path, not time.
-                //     //  (It might result in the prediction being wrong, but it is better than the user action being completely removed).
-                //     callPath = { ...callPath, runAtTime: getNextTime() };
-                //     retries++;
-                //     continue;
-                // }
-                //if (PathFunctionRunner.DEBUG_CALLS)
-                {
-                    console.log(`ERROR ${getDebugName(callPath, functionSpec, true)}`, e.stack);
-                }
-
-                // NOTE: commitFunction should never throw? If the server is dead, it still won't throw, so... we aren't try/catching it,
-                //  as I don't see how retrying would help (the same error would almost certainly happen again and again).
-                await proxyWatcher.commitFunction({
-                    canWrite: true,
-                    // NOTE: We don't set runAtTime, because the rejection might be due to the call being too old
-                    debugName: `error (${getDebugName(callPath, functionSpec)})`,
-                    eventWrite: true,
-                    doNotStoreWritesAsPredictions: true,
-                    watchFunction() {
-                        let syncedModule = functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId];
-                        syncedModule.Results[callPath.CallId] = atomicObjectWrite({
-                            error: e.stack || "",
-                            runCount: runCount,
-                            timeTaken: Date.now() - startTime,
-                            evalTime,
-                        });
-                    },
-                }, {
-                    onWritesCommitted(writes) {
-                        finalWrites = writes;
-                    },
-                });
-            }
-            if (isFunctionSpecOutdated) continue;
-            break;
-        }
-
-        if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callPath, functionSpec, true)}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
-        }
-
-        let wallTime = Date.now() - startTime;
-        let syncTime = wallTime - evalTime;
-
-
-        console.info("Finished FunctionRunner function", {
-            ...callPath, argsEncoded: "", functionSpec,
-            wallTime, syncTime, evalTime,
-            loops: runCount,
-        });
-        PathFunctionRunner.RUN_FINISH_COUNT++;
-    }
-}
-
-export async function preloadFunctions(specs: FunctionSpec[], progress?: DeployProgress) {
-    progress?.({ section: "Finding FunctionPreloadControllers", progress: 0 });
-    let nodeIds = await getControllerNodeIdList(FunctionPreloadController);
-    progress?.({ section: "Finding FunctionPreloadControllers", progress: 1 });
-    await Promise.allSettled(nodeIds.map(async nodeObj => {
-        let nodeId = nodeObj.nodeId;
-        let controller = FunctionPreloadController.nodes[nodeId];
-        let section = `${nodeObj.entryPoint}:${nodeId}|Preloading Functions`;
-        progress?.({ section, progress: 0 });
-        console.log(blue(`Preloading functions on ${String(nodeId)}`));
-        await errorToUndefined(controller.preloadFunctions(specs));
-        progress?.({ section, progress: 1 });
-        console.log(blue(`Finished preloading functions on ${String(nodeId)}`));
-    }));
-}
-
-export function getDevFunctionSpecFromCall(call: {
-    DomainName: string;
-    ModuleId: string;
-    FunctionId: string;
-}): FunctionSpec | undefined {
-    let domainName = call.DomainName;
-    let moduleId = call.ModuleId;
-    let functionId = call.FunctionId;
-
-    let devModule = getDevelopmentModule(moduleId);
-    if (!devModule) {
-        console.warn(yellow(`Cannot call ${domainName}.${moduleId}.Sources.${functionId} because the module is not deployed and not referenced in deploy.ts`));
-        return undefined;
-    }
-    let filePath = getModuleRelativePath(devModule);
-    let gitURL = getGitURLSync(undefined);
-    let gitRef = getGitRefSync(undefined);
-    return {
-        DomainName: domainName,
-        ModuleId: moduleId,
-        FunctionId: functionId,
-        exportPathStr: getExportPath(functionId),
-        FilePath: filePath,
-        gitURL,
-        gitRef,
-    };
-}
-
-class FunctionPreloaderBase {
-    async preloadFunctions(specs: FunctionSpec[]) {
-        if (isLocal()) return;
-        for (let spec of specs) {
-            await getModuleFromSpec(spec);
-        }
-    }
-}
-
-const FunctionPreloadController = SocketFunction.register(
-    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
-    new FunctionPreloaderBase(),
-    () => ({
-        preloadFunctions: {},
-    }),
-    () => ({
-        hooks: [requiresNetworkTrustHook],
-    }),
-    {
-        noAutoExpose: true,
-    }
-);
+import { delay, runInfinitePoll } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
+import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
+import { timeInHour, timeInSecond } from "socket-function/src/misc";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { registerDynamicResource } from "../diagnostics/trackResources";
+import { errorToUndefined, logErrors } from "../errors";
+import { getPathFromStr, getPathIndex } from "../path";
+import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+import { atomicObjectRead, atomicObjectWrite, doProxyOptions, isProxyBlockedByOrder, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { authorityStorage, compareTime, debugTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { getModuleFromSpec } from "./pathFunctionLoader";
+import debugbreak from "debugbreak";
+import { parseArgs } from "./PathFunctionHelpers";
+import { FunctionMetadata, PERMISSIONS_FUNCTION_ID, addRoutingPrefixForDeploy, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
+import { formatTime } from "socket-function/src/formatting/format";
+import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getDomain, isPublic } from "../config";
+import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
+import type { DeployProgress } from "../4-deploy/deployFunctions";
+import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
+setImmediate(() => import("../4-querysub/Querysub"));
+
+export const functionSchema = rawSchema<{
+    [domainName: string]: {
+        PathFunctionRunner: {
+            [ModuleId: string]: {
+                Data: unknown;
+                Sources: { [FunctionId: string]: FunctionSpec | undefined; };
+                Calls: { [CallId: string]: CallSpec | undefined; };
+                Results: { [CallId: string]: FunctionResult | undefined; };
+            };
+        };
+    };
+}>();
+void Promise.resolve().then(() => {
+    for (let moduleId of getAllDevelopmentModulesIds()) {
+        addRoutingPrefixForDeploy(getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Calls));
+        addRoutingPrefixForDeploy(getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Results));
+    }
+});
+
+export function commitCall(call: CallSpec) {
+    let domainName = call.DomainName;
+    let moduleId = call.ModuleId;
+    let callId = call.CallId;
+    proxyWatcher.writeOnly({
+        canWrite: true,
+        eventWrite: true,
+        doNotStoreWritesAsPredictions: true,
+        // NOTE: We write the call in the present, it is callSpec.runAtTime that is in the past. No one cares
+        //  when the call was added, only when it wants to run at.
+        watchFunction: function writeCall() {
+            functionSchema()[domainName].PathFunctionRunner[moduleId].Calls[callId] = atomicObjectWrite(call);
+        },
+    });
+}
+
+export const DEPTH_TO_DATA = 4;
+export const DOMAIN_INDEX = 0;
+export const MODULE_INDEX = 2;
+
+export function getSchemaPartsFromPath(path: string): { domainName: string; moduleId: string } | undefined {
+    let parts = getPathFromStr(path);
+    let domainName = parts[0];
+    let moduleId = parts[2];
+    if (!domainName || !moduleId) return undefined;
+    return { domainName, moduleId };
+}
+
+export type FunctionSpec = {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+    exportPathStr: string;
+    FilePath: string;
+    gitURL: string;
+    /** Ex, the hash */
+    gitRef: string;
+
+    maxLocksOverride?: number;
+};
+
+export interface CallSpec {
+    DomainName: string;
+    ModuleId: string;
+    CallId: string;
+    FunctionId: string;
+    // NOTE: This is no longer JSON. Use parseArgs to decode it
+    argsEncoded: string;
+    callerMachineId: string;
+    callerIP: string;
+    runAtTime: Time;
+
+    // Not just used for debugging, also used to add special proxy-related warnings. 
+    fromProxy?: string;
+
+    filterable?: Filterable;
+}
+export function debugCallSpec(spec: CallSpec): string {
+    return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
+}
+export type FunctionResult = ({
+
+} | {
+    error: string;
+}) & {
+    timeTaken: number;
+    evalTime: number;
+    lastInternalLoopCount: number;
+    outerLoopCount: number;
+    totalInternalLoopCount: number;
+    totalTime: number;
+};
+
+let currentCallSpec: { spec: CallSpec; fnc: FunctionSpec; } | undefined;
+export function getCurrentCall() {
+    if (!currentCallSpec) {
+        debugbreak(2);
+        debugger;
+        throw new Error("Not presently in a call, and so cannot get call");
+    }
+    return currentCallSpec.spec;
+}
+export function getCurrentCallAllowUndefined() {
+    return currentCallSpec?.spec;
+}
+export function getCurrentCallObj() {
+    return currentCallSpec;
+}
+/** NOTE: We require the FunctionSpec so we know what has is loaded, to identify the
+ *      correct schema. This should be available, but if it isn't... we MIGHT be able
+ *      to work around it, if we can be assured the schema won't be used?
+ *      - This is so pre-loading of functions (which might happen without a deploy, if the deploy script
+ *          is killed part of the way through), doesn't change schemas until the deploy actually
+ *          updates the underlying functions.
+  */
+export function overrideCurrentCall<T>(config: { spec: CallSpec; fnc: FunctionSpec; }, code: () => T) {
+    let prev = currentCallSpec;
+    currentCallSpec = config;
+    try {
+        return code();
+    } finally {
+        currentCallSpec = prev;
+    }
+}
+
+function getDebugName(call: CallSpec, functionSpec: FunctionSpec | undefined, colored = false) {
+    if (!functionSpec) {
+        return `${call.DomainName}:${call.ModuleId}|${call.FunctionId}|${debugTime(call.runAtTime)}`;
+    }
+    let pathName = getPathFromStr(functionSpec.exportPathStr).slice(1).join(".");
+    let mainPart = `${call.DomainName}:${functionSpec.FilePath}|${pathName}|${debugTime(call.runAtTime)}`;
+    if (colored) {
+        return `${magenta(mainPart)}`;
+    } else {
+        return `${mainPart}`;
+    }
+}
+
+type PermissionsCheckType = {
+    new(callerMachineId: { callerMachineId: string; callerIP: string; }): {
+        checkPermissions(path: string): { permissionsPath: string; allowed: boolean; };
+    };
+    skipPermissionsChecks: <T>(code: () => T) => T;
+}
+
+export type DebugFunctionShardInfo = {
+    domainName: string;
+    shardRange: { startFraction: number, endFraction: number };
+    secondaryShardRange?: { startFraction: number, endFraction: number };
+};
+let debugFunctionRunnerShards: DebugFunctionShardInfo[] = [];
+export function debug_getFunctionRunnerShards() {
+    return debugFunctionRunnerShards;
+}
+setImmediate(() => {
+    set_debug_getFunctionRunnerShards(debug_getFunctionRunnerShards);
+});
+
+type CallStats = {
+    outerLoopCount: number;
+    totalInternalLoopCount: number;
+    lastInternalLoopCount: number;
+    isInsideRunCall: boolean;
+    timeTaken: number;
+    evalTime: number;
+    firstQueuedTime: number;
+};
+
+export class PathFunctionRunner {
+    public static RUN_START_COUNT = 0;
+    public static RUN_FINISH_COUNT = 0;
+    public static DEBUG_CALLS = false;
+    public static DEBUG_CALL_TRIGGERS = false;
+    public static DEBUG_WATCHES_THRESHOLD = 5;
+    public static MAX_WATCH_LOOPS = 1000;
+
+    constructor(private config: {
+        domainName: string;
+        shardRange: { startFraction: number, endFraction: number };
+        secondaryShardRange?: { startFraction: number, endFraction: number };
+        PermissionsChecker: PermissionsCheckType | undefined;
+        filterSelector?: FilterSelector;
+    }) {
+        SocketFunction.expose(FunctionPreloadController);
+        debugFunctionRunnerShards.push({
+            domainName: config.domainName,
+            shardRange: config.shardRange,
+            secondaryShardRange: config.secondaryShardRange,
+        });
+        logErrors(this.startWatching());
+    }
+
+    // Calls runCall on any calls that match our domain + shard range
+    private async startWatching() {
+        const { Querysub } = await import("../4-querysub/Querysub");
+        let { shardRange, secondaryShardRange } = this.config;
+
+        // We will use PathValueProxyWatcher to watch the paths
+        // Maintain only one outstanding runCall per callId
+        //  - Once it finishes, we can trigger another call, if we notice the result is not set.
+        //      - AS we should get write prediction for the result, we can rerun our proxy check
+        //          loop immediately.
+
+        let fullFraction = shardRange;
+        if (secondaryShardRange) {
+            fullFraction = {
+                startFraction: Math.min(shardRange.startFraction, secondaryShardRange.startFraction),
+                endFraction: Math.max(shardRange.endFraction, secondaryShardRange.endFraction),
+            };
+        }
+
+        let config = this.config;
+
+        let self = this;
+
+        let outstandingCalls = 0;
+
+        let watchModuleCalls = cache((moduleId: string) => {
+            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls);
+            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results);
+            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, fullFraction);
+            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, fullFraction);
+        });
+
+        let runningCalls = new Set<string>();
+
+        // Rarely we might need to queue a function multiple times, when we are late to receive rejections.
+        //  HOWEVER, after enough times, we should stop, as we will probably just infinitely queue it.
+        // For debugging, we should never really be having to re-queue it. And if we requeue it a lot, it makes the logs larger, which makes it harder to debug it! 
+        const MAX_QUEUE_COUNT = isPublic() ? 100 : 14;
+        let queueLimitCounts = new Map<string, number>();
+        // Clear every hour, so we don't leave
+        runInfinitePoll(timeInHour, () => queueLimitCounts.clear());
+
+        const domainName = config.domainName;
+        const watcher = proxyWatcher.createWatcher({
+            trackTriggers: true,
+            skipPermissionsCheck: true,
+            hasSideEffects: true,
+            // Maybe allow running with unsynced reads? This might be good, because we don't need all reads
+            //  to detect calls. BUT, it might also be bad, because it might cause us to rerun in a cascading
+            //  fashion? (As in, be really slow).
+            //  - Shouldn't be needed, because writes don't take that long
+            //  - Could fix bugs where we never get a read, and therefore all function calls fails to run?
+            // allowUnsyncedReads: true
+            watchFunction: function findFunctionsToCall() {
+                // TODO: Maybe abstract the "preserve watchers" stuff,
+                //  so that it is automatically done at the start of our trigger.
+                const watcher = proxyWatcher.getTriggeredWatcher();
+
+                // Preserve previous watches explicitly (so we just do a diff, instead of accessing every
+                //  value every time).
+                proxyWatcher.reuseLastWatches();
+
+                let data = () => functionSchema()[domainName].PathFunctionRunner;
+
+                let moduleIds = Object.keys(data());
+                if (!isPublic()) {
+                    moduleIds.push(...getAllDevelopmentModulesIds());
+                    moduleIds = Array.from(new Set(moduleIds));
+                }
+
+                for (let moduleId of moduleIds) {
+                    watchModuleCalls(moduleId);
+                }
+
+                let newCalls: { call: CallSpec; functionSpec: FunctionSpec }[] = [];
+
+                if (watcher.triggeredByChanges) {
+                    for (let path of watcher.triggeredByChanges.paths) {
+                        let callsOrResult = getPathIndex(path, 3);
+                        if (callsOrResult !== "Calls" && callsOrResult !== "Results") continue;
+                        if (getPathIndex(path, 1) !== "PathFunctionRunner") continue;
+                        if (getPathIndex(path, 0) !== domainName) continue;
+
+                        const callId = getPathIndex(path, 4);
+                        if (!callId) continue;
+                        let moduleId = getPathIndex(path, 2);
+                        if (!moduleId) continue;
+                        if (!moduleIds.includes(moduleId)) continue;
+
+                        let moduleData = data()[moduleId];
+
+                        let result = atomicObjectRead(moduleData.Results[callId]);
+                        if (result) continue;
+
+                        let callData = atomicObjectRead(moduleData.Calls[callId]);
+                        if (!callData) continue;
+
+                        // Ignore invalid data
+                        if (callData.DomainName !== domainName) continue;
+                        if (callData.ModuleId !== moduleId) continue;
+                        if (callData.CallId !== callId) continue;
+
+                        const functionId = callData.FunctionId;
+                        let functionSpec = atomicObjectRead(moduleData.Sources[functionId]);
+                        if (!functionSpec) {
+                            if (!isSynced(moduleData.Sources[callData.FunctionId])) {
+                                continue;
+                            }
+                            if (!isPublic()) {
+                                functionSpec = getDevFunctionSpecFromCall(callData);
+                                if (!functionSpec) continue;
+                            } else {
+                                console.warn(yellow(`Cannot call ${callData.DomainName}.${callData.ModuleId}.Sources.${callData.FunctionId} because it is not deployed`));
+                                continue;
+                            }
+                        }
+
+                        // If we haven't synced the result, we can't know if it is ready or not
+                        //  (we'll run anyways once it is synced). Most other parts are immutable,
+                        //  but the result is by definition not, so... it is one of the values we check!
+                        if (!isSynced(moduleData.Results[callId])) continue;
+
+                        if (runningCalls.has(callId)) continue;
+                        runningCalls.add(callId);
+
+                        let limitCount = queueLimitCounts.get(callId) || 0;
+                        limitCount++;
+                        queueLimitCounts.set(callId, limitCount);
+                        // NOTE: Calls are event writes, so... they should just clean themselves up, after we ignore them for long enough.
+                        if (limitCount >= MAX_QUEUE_COUNT) {
+                            // Only error the first time, as we don't need need that many errors
+                            if (limitCount === MAX_QUEUE_COUNT) {
+                                console.error(`(Likely a missing synced value) Tried to requeue a function to run too many times (${limitCount}) for ${getDebugName(callData, functionSpec, true)}. This is NOT due to cascading reads. It might be due to repeated rejections?`, {
+                                    callId
+                                });
+
+                                let stats = self.callStats.get(callId);
+                                void self.finishCallWithError({
+                                    callSpec: callData,
+                                    functionSpec,
+                                    error: `Tried to requeue a function to run too many times (${limitCount}) for ${getDebugName(callData, functionSpec, true)}. This is NOT due to cascading reads. It might be due to repeated rejections?`,
+                                    runCount: stats?.lastInternalLoopCount || 0,
+                                    startTime: Date.now(),
+                                    evalTime: 0,
+                                    outerLoopCount: stats?.outerLoopCount || 0,
+                                    totalInternalLoopCount: stats?.totalInternalLoopCount || 0,
+                                });
+                            }
+                            continue;
+                        }
+
+                        if (PathFunctionRunner.DEBUG_CALLS) {
+                            console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
+                            let resultsPath = getProxyPath(() => moduleData.Results[callId]);
+                            let history = authorityStorage.getValuePlusHistory(resultsPath);
+                            console.log(`    History: ${history.length}`);
+                            for (let { valid, time, canGCValue } of history) {
+                                console.log(`        ${valid ? "✅" : "❌"} ${debugTime(time)} ${canGCValue ? "value is undefined" : ""}`);
+                            }
+                        }
+                        newCalls.push({ call: callData, functionSpec });
+                    }
+                }
+
+                // Better run first calls first, or else we will just pointlessly cause rejections.
+                newCalls.sort((a, b) => compareTime(a.call.runAtTime, b.call.runAtTime));
+                for (let { call, functionSpec } of newCalls) {
+                    // We don't need wait for all pending synced accesses to finish.
+                    //  Worst case we have an old functionSpec, and then runCall has to loop again.
+                    //  Otherwise the call request is (or should be) atomic, so it this should just work...
+                    //  - Or if it is rejected, we will detect it has no result, and run it again.
+                    // NOTE: We run all calls in parallel, so we can load data in parallel. This is a bit dangerous, and might
+                    //      result in out of order execution, but... we commit at runTimes, so, this should eventually
+                    //      resolve itself due to rejections (which will change the call result, which SHOULD cause us to rerun again).
+                    //      - This is a bit dangerous, as we might overwhelm ourself. We should address this one we support
+                    //          FunctionRunner sharding.
+                    outstandingCalls++;
+                    let runCallPromise = self.runCall(call, functionSpec);
+                    void runCallPromise.finally(() => {
+                        outstandingCalls--;
+                        // IMPORTANT! We remove the call, NOT for GC, but! Because if the call writes are rejected (due to contention), then the Result will also be rejected, changing it back to undefined, so (as long as little enough time has passed), we will run the call again! Or rather, we will try to, as long as runningCalls doesn't have the value!
+                        runningCalls.delete(call.CallId);
+                    });
+                    logErrors(runCallPromise);
+                }
+            },
+        });
+        registerDynamicResource("paths|PathFunctionRunner.watches", () => watcher.lastWatches.paths.size);
+        registerDynamicResource("paths|PathFunctionRunner.outstandingCalls", () => outstandingCalls);
+
+        // If local, periodically check all modules
+        //  NOTE: `yarn deploy` commits the moduleIds to the database, but calls do not, as this would require either an extra
+        //      read, or an extra write. So this is somewhat a hack, but also fairly safe.
+        if (!isPublic()) {
+            let prevModuleIds = new Set<string>();
+            runInfinitePoll(timeInSecond * 15, () => {
+                let moduleIds = getAllDevelopmentModulesIds();
+                let changed = false;
+                for (let moduleId of moduleIds) {
+                    if (!prevModuleIds.has(moduleId)) {
+                        changed = true;
+                        break;
+                    }
+                }
+                prevModuleIds = new Set(moduleIds);
+                if (changed) {
+                    watcher.explicitlyTrigger();
+                }
+            });
+        }
+    }
+
+
+    @measureFnc
+    private async getExportsFromSpec(spec: FunctionSpec): Promise<unknown> {
+        let module = await getModuleFromSpec(spec);
+        return module.exports;
+    }
+
+    private getSecondaryShardDelay(callPath: CallSpec): number {
+        let { shardRange, secondaryShardRange } = this.config;
+        if (!secondaryShardRange) return 0;
+        let fraction = getRoutingOverridePart(callPath.CallId)?.route;
+        if (fraction === undefined) throw new Error(`No routing override found for callId, so we can't determine if we should handle it. CallId: ${callPath.CallId}`);
+        // It isn't secondary if it is primary
+        if (shardRange.startFraction <= fraction && fraction < shardRange.endFraction) return 0;
+        if (!(secondaryShardRange.startFraction <= fraction && fraction < secondaryShardRange.endFraction)) return 0;
+        let fractionDistanceToPrimary = Math.min(
+            Math.abs(fraction - shardRange.startFraction),
+            Math.abs(fraction - shardRange.endFraction),
+        );
+        return Math.min(
+            // No more than 50% of accepted change, otherwise it might not be accepted
+            MAX_ACCEPTED_CHANGE_AGE * 0.5,
+            // Default values in case MAX_ACCEPTED_CHANGE_AGE is VERY high (minutes, or even hours)
+            30_000 + 10_000 * fractionDistanceToPrimary,
+            // Wait 5000 seconds, then start 
+            5000 + MAX_ACCEPTED_CHANGE_AGE * fractionDistanceToPrimary * 0.05 / (shardRange.endFraction - shardRange.startFraction),
+        );
+    };
+
+    private callsRemainingInTick = cache((wait: true) => {
+        // NOTE: This gives us 100K calls per second, which is more than we can run anyways.
+        //  We just need to wait a bit, otherwise the IO loop can never run, and so our finished calls
+        //  can never actually finish, until eventually they are too late to finish and get rejected!
+        let tickDone = delay(10);
+        void tickDone.finally(() => {
+            this.callsRemainingInTick.clear(true);
+        });
+        return { calls: 1000, tickDone };
+    });
+    private async letIOClear(): Promise<void> {
+        while (true) {
+            let callsObj = this.callsRemainingInTick(true);
+            callsObj.calls--;
+            if (callsObj.calls >= 0) return;
+            await callsObj.tickDone;
+        }
+    }
+
+    private callStats = new Map<string, CallStats>();
+
+    private async runCall(callSpec: CallSpec, functionSpec: FunctionSpec): Promise<void> {
+        let callId = callSpec.CallId;
+        let stats = this.callStats.get(callId);
+        if (!stats) {
+            // HACK: Just completely reset the map after it gets big enough. This prevents us from crashing due to the map getting too large, as there's a limit to how large maps can get. 
+            if (this.callStats.size > 1000 * 1000) {
+                this.callStats.clear();
+            }
+            stats = {
+                outerLoopCount: 0,
+                totalInternalLoopCount: 0,
+                lastInternalLoopCount: 0,
+                isInsideRunCall: false,
+                timeTaken: 0,
+                evalTime: 0,
+                firstQueuedTime: Date.now(),
+            };
+            this.callStats.set(callId, stats);
+        }
+
+        if (stats.isInsideRunCall) {
+            console.error(`Attempted to re-enter runCall for ${callId} while already inside runCall`);
+            return;
+        }
+
+        stats.isInsideRunCall = true;
+        stats.outerLoopCount++;
+        try {
+            await this.runCallBase(callSpec, functionSpec, stats);
+        } finally {
+            stats.isInsideRunCall = false;
+        }
+    }
+
+    private async runCallBase(callSpec: CallSpec, functionSpec: FunctionSpec, stats: CallStats): Promise<void> {
+        const { Querysub } = await import("../4-querysub/Querysub");
+
+        const PermissionsChecker = this.config.PermissionsChecker;
+        let skipPermissions = <T>(code: () => T) => code();
+        if (PermissionsChecker) {
+            skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
+        }
+
+        if (!doesMatch(callSpec.filterable, this.config.filterSelector)) {
+            return;
+        }
+
+        console.info(`New function call: ${getDebugName(callSpec, functionSpec, true)}`, {
+            callId: callSpec.CallId,
+            timeId: callSpec.runAtTime.time,
+            functionId: callSpec.FunctionId,
+            moduleId: callSpec.ModuleId,
+            gitRef: functionSpec.gitRef,
+        });
+
+        await this.letIOClear();
+
+        let secondaryDelay = this.getSecondaryShardDelay(callSpec);
+        if (secondaryDelay) {
+            async function getIsFinished() {
+                return await proxyWatcher.commitFunction({
+                    watchFunction() {
+                        let syncedModule = skipPermissions(() =>
+                            functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId]
+                        );
+                        return !!atomicObjectRead(syncedModule.Results[callSpec.CallId]);
+                    },
+                });
+            }
+            // Call once, so we are already watching the 
+            await delay(secondaryDelay);
+            let isFinished = await getIsFinished();
+            if (isFinished) {
+                console.info(`Function run finished before we could run it (another server must have ran it)`, {
+                    callId: callSpec.CallId,
+                });
+                return;
+            }
+            if (PathFunctionRunner.DEBUG_CALLS) {
+                let fraction = getRoutingOverridePart(callSpec.CallId)?.route;
+                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction?.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${getDebugName(callSpec, functionSpec, true)}`);
+            }
+        }
+
+        PathFunctionRunner.RUN_START_COUNT++;
+        if (PathFunctionRunner.DEBUG_CALLS) {
+            console.log(`STARTING ${getDebugName(callSpec, functionSpec, true)}`);
+        }
+        let startTime = Date.now();
+        let runCount = 0;
+        stats.lastInternalLoopCount = 0;
+
+        let finalWrites: PathValue[] | undefined;
+
+        let evalTime = 0;
+        let retries = 0;
+
+        let nooped = false;
+
+        while (true) {
+            let isFunctionSpecOutdated = false;
+            try {
+                if (retries > 10) {
+                    console.error(`Could not sync stable function spec, aborting call. Retries: ${retries}`, {
+                        callId: callSpec.CallId,
+                    });
+                    throw new Error(`Could not sync stable function spec, aborting call. Retries: ${retries}`);
+                }
+
+                let moduleExports = await this.getExportsFromSpec(functionSpec);
+
+                // ALWAYS get the permissions as well. This isn't perfect, as we could very well access data in other files, but... it helps
+                //  reduce extra iterations inside of commitFunction...
+                await getModuleFromSpec({ ...functionSpec, exportPathStr: getExportPath(PERMISSIONS_FUNCTION_ID), FunctionId: PERMISSIONS_FUNCTION_ID, });
+
+                let exportPath = getPathFromStr(functionSpec.exportPathStr);
+                let exportObj = moduleExports as any;
+                for (let path of exportPath) {
+                    exportObj = exportObj[path];
+                }
+                let baseFunction = exportObj as Function;
+
+                if (typeof baseFunction !== "function") {
+                    throw new Error(`Export at ${JSON.stringify(exportPath.join("."))} was not a function (it was ${typeof baseFunction}). If in development, you must restart the development function runner server when you add new functions.`);
+                }
+
+                let devFunctionMetadata: FunctionMetadata | undefined;
+
+                let devModule = getDevelopmentModule(functionSpec.ModuleId);
+                if (devModule) {
+                    let schemaObj = getSchemaObject(devModule);
+                    devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
+                }
+
+                console.info(`Running function: ${getDebugName(callSpec, functionSpec, true)}`, {
+                    callId: callSpec.CallId,
+                    outerLoop: retries,
+                });
+
+                await proxyWatcher.commitFunction({
+                    canWrite: true,
+                    debugName: getDebugName(callSpec, functionSpec),
+                    runAtTime: callSpec.runAtTime,
+                    getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callSpec)),
+                    nestedCalls: "inline",
+                    temporary: true,
+                    maxLocksOverride: devFunctionMetadata ? devFunctionMetadata.maxLocksOverride : functionSpec.maxLocksOverride,
+                    source: callSpec.CallId,
+                    watchFunction: function runCallWatcher() {
+                        runCount++;
+                        stats.lastInternalLoopCount = runCount;
+                        stats.totalInternalLoopCount++;
+                        if (PathFunctionRunner.DEBUG_CALLS) {
+                            console.log(`Evaluating (try count ${runCount}) ${getDebugName(callSpec, functionSpec, true)}`);
+                        }
+                        if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
+                            let errorMessage = `MAX_WATCH_LOOPS exceeded for ${getDebugName(callSpec, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`;
+                            console.error(errorMessage, {
+                                callId: callSpec.CallId,
+                                runCount,
+                                limit: PathFunctionRunner.MAX_WATCH_LOOPS,
+                            });
+                            throw new Error(errorMessage);
+                        }
+
+                        // We NEED to depend on the function spec, so spec updates can be atomic (which is needed to deploy an update
+                        //  to many nodes at once, without having any period where multiple versions both commit values).
+                        let syncedModule = skipPermissions(() =>
+                            functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId]
+                        );
+
+                        // Make sure we are running the latest function (by checking it here, we also lock the values,
+                        //  so if it is updated we atomically rerun / reject all calls which were racing with us).
+                        if (!!isPublic()) {
+                            let syncedSpec = skipPermissions(() =>
+                                atomicObjectRead(syncedModule.Sources[callSpec.FunctionId])
+                            );
+                            if (!syncedSpec) {
+                                throw new Error(`Function spec not found for ${getDebugName(callSpec, functionSpec, true)}`);
+                            }
+
+                            //  (We also need to depend on the RIGHT function spec).
+                            if (
+                                syncedSpec && (
+                                    syncedSpec.DomainName !== functionSpec.DomainName
+                                    || syncedSpec.ModuleId !== functionSpec.ModuleId
+                                    || syncedSpec.FilePath !== functionSpec.FilePath
+                                    || syncedSpec.FunctionId !== functionSpec.FunctionId
+                                    || syncedSpec.exportPathStr !== functionSpec.exportPathStr
+                                    || syncedSpec.FilePath !== functionSpec.FilePath
+                                    || syncedSpec.gitRef !== functionSpec.gitRef
+                                    || syncedSpec.gitURL !== functionSpec.gitURL
+                                    || syncedSpec.maxLocksOverride !== functionSpec.maxLocksOverride
+                                )
+                            ) {
+                                isFunctionSpecOutdated = true;
+                                functionSpec = {
+                                    DomainName: syncedSpec.DomainName,
+                                    ModuleId: syncedSpec.ModuleId,
+                                    FilePath: syncedSpec.FilePath,
+                                    FunctionId: syncedSpec.FunctionId,
+                                    exportPathStr: syncedSpec.exportPathStr,
+                                    gitRef: syncedSpec.gitRef,
+                                    gitURL: syncedSpec.gitURL,
+                                };
+                                return;
+                            }
+                        }
+
+                        let evalTimeStart = Date.now();
+                        try {
+                            let args = parseArgs(callSpec);
+                            overrideCurrentCall({ spec: callSpec, fnc: functionSpec, }, () => {
+                                baseFunction(...args);
+                            });
+                        } finally {
+                            let evalTimeDelta = Date.now() - evalTimeStart;
+                            evalTime += evalTimeDelta;
+                            stats.evalTime += evalTimeDelta;
+                        }
+
+                        // NOTE: The results are only temporarily stored, but serve as a way to let clients
+                        //  know figure out the dependencies for the call, which makes function call prediction
+                        //  possible.
+                        skipPermissions(() => {
+                            let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callSpec.CallId]));
+                            if (result) {
+                                nooped = true;
+                                // It is important to NOOP if it has already been written. Otherwise there could be multiple
+                                //  runs of the same function, which could easily vary (if they use Math.random(), etc).
+                                //  WHICH, as the write time is predetermined, would cause ValuePaths with equal times, but
+                                //  different values (which breaks a lot of assumptions much of the code depends on! such
+                                //  as locks, etc).
+                                if (!secondaryDelay && Querysub.isAllSynced()) {
+                                    console.warn(`Skipping function write, as it has already been written to by another FunctionRunner.`);
+                                }
+                                return;
+                            }
+                            nooped = false;
+                            let currentTimeTaken = Date.now() - startTime;
+                            stats.timeTaken = currentTimeTaken;
+                            let totalTime = Date.now() - stats.firstQueuedTime;
+                            syncedModule.Results[callSpec.CallId] = atomicObjectWrite({
+                                lastInternalLoopCount: runCount,
+                                outerLoopCount: stats.outerLoopCount,
+                                totalInternalLoopCount: stats.totalInternalLoopCount,
+                                timeTaken: currentTimeTaken,
+                                evalTime,
+                                totalTime,
+                            });
+                            proxyWatcher.setEventPath(() => syncedModule.Results[callSpec.CallId]);
+                        });
+
+                        let watcher = proxyWatcher.getTriggeredWatcher();
+                        if (watcher.hasAnyUnsyncedAccesses()) {
+                            console.info(`Function run, unsynced values`, {
+                                callId: callSpec.CallId,
+                                count: watcher.pendingUnsyncedAccesses.size,
+                                countParents: watcher.pendingUnsyncedParentAccesses.size,
+                                promiseUnsynced: watcher.specialPromiseUnsynced,
+                                blockedByOrder: isProxyBlockedByOrder(watcher),
+                                path: Array.from(watcher.pendingUnsyncedAccesses)[0],
+                            });
+                        }
+                    },
+                }, {
+                    onWritesCommitted(writes) {
+                        finalWrites = writes;
+                    },
+                });
+            } catch (e: any) {
+
+                retries++;
+                if (isFunctionSpecOutdated) continue;
+
+                // NOTE: We are changing our paradigm to reject early, rather than retry. This is more stable. You can't really commit
+                //  important values and then just leave, unless those values have no locks. This helps prevent values from being
+                //  in different states on different machines, by just rejecting the call.
+                // if (e.message.includes("MAX_CHANGE_AGE_EXCEEDED") && retries < 10 && evalTime < MAX_ACCEPTED_CHANGE_AGE) {
+                //     // NOTE: This should never really happen. It is only really if our PathValueServer goes down for a minute or so,
+                //     //  so we don't just drop all of our function calls.
+                //     //  - This is why we log every time, because no user actions should be able to result in this happening frequently.
+                //     console.error(red(`RETRYING due to MAX_CHANGE_AGE_EXCEEDED. Hopefully the server will be more responsive this time (retry ${retries}): ${getDebugName(callPath, functionSpec, true)}`));
+                //     // NOTE: Updating the time is fine, and won't break prediction rejections, as we reject based on path, not time.
+                //     //  (It might result in the prediction being wrong, but it is better than the user action being completely removed).
+                //     callPath = { ...callPath, runAtTime: getNextTime() };
+                //     retries++;
+                //     continue;
+                // }
+                //if (PathFunctionRunner.DEBUG_CALLS)
+                {
+                    console.log(`Function run error ${getDebugName(callSpec, functionSpec, true)}`, {
+                        callId: callSpec.CallId,
+                        error: e.stack,
+                    });
+                }
+
+                finalWrites = await this.finishCallWithError({
+                    callSpec,
+                    functionSpec,
+                    error: e.stack || "",
+                    runCount,
+                    startTime,
+                    evalTime,
+                    outerLoopCount: stats.outerLoopCount,
+                    totalInternalLoopCount: stats.totalInternalLoopCount,
+                });
+            }
+
+            retries++;
+            if (isFunctionSpecOutdated) continue;
+            break;
+        }
+
+        if (PathFunctionRunner.DEBUG_CALLS) {
+            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callSpec, functionSpec, true)}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
+        }
+
+        let wallTime = Date.now() - startTime;
+        let syncTime = wallTime - evalTime;
+
+        for (let write of finalWrites || []) {
+            console.info(`Function path write`, {
+                callId: callSpec.CallId,
+                path: write.path,
+                timeId: write.time.time,
+                isTransparent: write.isTransparent,
+            });
+        }
+
+        console.info(`Finished function evaluation: ${getDebugName(callSpec, functionSpec, true)}`, {
+            callId: callSpec.CallId,
+            outerLoop: retries,
+            finalWrites: finalWrites?.length,
+            argsEncoded: "", functionSpec,
+            wallTime, syncTime, evalTime,
+            loops: runCount,
+        });
+        PathFunctionRunner.RUN_FINISH_COUNT++;
+    }
+
+    // NOTE: This does and a 
+    public async finishCallWithError(config: {
+        callSpec: CallSpec;
+        functionSpec: FunctionSpec | undefined;
+        error: string;
+        runCount: number;
+        startTime: number;
+        evalTime: number;
+        outerLoopCount: number;
+        totalInternalLoopCount: number;
+    }): Promise<PathValue[] | undefined> {
+
+        const { Querysub } = await import("../4-querysub/Querysub");
+        if (Querysub.isInSyncedCall()) {
+            Querysub.onCommitFinished(() => {
+                void this.finishCallWithError(config);
+            });
+        }
+
+        let { callSpec, functionSpec, error, runCount, startTime, evalTime, outerLoopCount, totalInternalLoopCount } = config;
+
+        let finalWrites: PathValue[] | undefined;
+
+        let callStats = this.callStats.get(callSpec.CallId);
+        let totalTime = callStats ? Date.now() - callStats.firstQueuedTime : Date.now() - startTime;
+
+        await proxyWatcher.commitFunction({
+            canWrite: true,
+            // NOTE: We don't set runAtTime, because the rejection might be due to the call being too old
+            debugName: `error (${getDebugName(callSpec, functionSpec)})`,
+            eventWrite: true,
+            doNotStoreWritesAsPredictions: true,
+            watchFunction() {
+                let syncedModule = functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId];
+                // Don't clobber the value. I'm pretty sure event writes still read values, so we should still do a synchronization to see if there is an existing result. 
+                let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callSpec.CallId]));
+                if (result) return;
+                syncedModule.Results[callSpec.CallId] = atomicObjectWrite({
+                    error,
+                    lastInternalLoopCount: runCount,
+                    outerLoopCount,
+                    totalInternalLoopCount,
+                    timeTaken: Date.now() - startTime,
+                    evalTime,
+                    totalTime,
+                });
+            },
+        }, {
+            onWritesCommitted(writes) {
+                finalWrites = writes;
+            },
+        });
+        return finalWrites;
+    }
+}
+
+export async function preloadFunctions(specs: FunctionSpec[], progress?: DeployProgress) {
+    progress?.({ section: "Finding FunctionPreloadControllers", progress: 0 });
+    let nodeIds = await getControllerNodeIdList(FunctionPreloadController);
+    progress?.({ section: "Finding FunctionPreloadControllers", progress: 1 });
+    await Promise.allSettled(nodeIds.map(async nodeObj => {
+        let nodeId = nodeObj.nodeId;
+        let controller = FunctionPreloadController.nodes[nodeId];
+        let section = `${nodeObj.entryPoint}:${nodeId}|Preloading Functions`;
+        progress?.({ section, progress: 0 });
+        console.log(blue(`Preloading functions on ${String(nodeId)}`));
+        await errorToUndefined(controller.preloadFunctions(specs));
+        progress?.({ section, progress: 1 });
+        console.log(blue(`Finished preloading functions on ${String(nodeId)}`));
+    }));
+}
+
+export function getDevFunctionSpecFromCall(call: {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+}): FunctionSpec | undefined {
+    let domainName = call.DomainName;
+    let moduleId = call.ModuleId;
+    let functionId = call.FunctionId;
+
+    let devModule = getDevelopmentModule(moduleId);
+    if (!devModule) {
+        console.warn(yellow(`Cannot call ${domainName}.${moduleId}.Sources.${functionId} because the module is not deployed and not referenced in deploy.ts`));
+        return undefined;
+    }
+    let filePath = getModuleRelativePath(devModule);
+    let gitURL = getGitURLSync(undefined);
+    let gitRef = getGitRefSync(undefined);
+    return {
+        DomainName: domainName,
+        ModuleId: moduleId,
+        FunctionId: functionId,
+        exportPathStr: getExportPath(functionId),
+        FilePath: filePath,
+        gitURL,
+        gitRef,
+    };
+}
+
+class FunctionPreloaderBase {
+    async preloadFunctions(specs: FunctionSpec[]) {
+        if (!isPublic()) return;
+        for (let spec of specs) {
+            await getModuleFromSpec(spec);
+        }
+    }
+}
+
+const FunctionPreloadController = SocketFunction.register(
+    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
+    new FunctionPreloaderBase(),
+    () => ({
+        preloadFunctions: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);