querysub 0.2.0

package/src/3-path-functions/PathFunctionRunner.ts

@@ -0,0 +1,619 @@
+import { delay, runInfinitePoll } from "socket-function/src/batching";
+import { cache } from "socket-function/src/caching";
+import { magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { sha256HashBuffer, timeInHour } from "socket-function/src/misc";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { getLowUint32, getShortNumber } from "../bits";
+import { registerDynamicResource } from "../diagnostics/trackResources";
+import { logErrors } from "../errors";
+import { getPathDepth, getPathFromStr, getPathIndex, trimPathStrToDepth } from "../path";
+import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+import { atomic, atomicObjectRead, atomicObjectWrite, doAtomicWrites, doProxyOptions, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { __getRoutingHash, authorityStorage, compareTime, debugTime, epochTime, getNextTime, isCoreQuiet, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { getModuleFromSpec, watchModuleHotreloads } from "./pathFunctionLoader";
+import debugbreak from "debugbreak";
+import { Querysub } from "../4-querysub/Querysub";
+import { parseArgs } from "./PathFunctionHelpers";
+import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
+import { formatTime } from "socket-function/src/formatting/format";
+import { set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+
+export const functionSchema = rawSchema<{
+    [domainName: string]: {
+        PathFunctionRunner: {
+            [ModuleId: string]: {
+                Data: unknown;
+                Sources: { [FunctionId: string]: FunctionSpec | undefined; };
+                Calls: { [CallId: string]: CallSpec | undefined; };
+                Results: { [CallId: string]: FunctionResult | undefined; };
+            };
+        };
+    };
+}>();
+
+export const DEPTH_TO_DATA = 4;
+export const DOMAIN_INDEX = 0;
+export const MODULE_INDEX = 2;
+
+export function getSchemaPartsFromPath(path: string): { domainName: string; moduleId: string } | undefined {
+    let parts = getPathFromStr(path);
+    let domainName = parts[0];
+    let moduleId = parts[2];
+    if (!domainName || !moduleId) return undefined;
+    return { domainName, moduleId };
+}
+
+export type FunctionSpec = {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+    exportPathStr: string;
+    FilePath: string;
+    gitURL: string;
+    gitRef: string;
+};
+
+export interface CallSpec {
+    DomainName: string;
+    ModuleId: string;
+    CallId: string;
+    FunctionId: string;
+    // NOTE: This is no longer JSON. Use parseArgs to decode it
+    argsEncoded: string;
+    callerMachineId: string;
+    callerIP: string;
+    runAtTime: Time;
+}
+export function debugCallSpec(spec: CallSpec): string {
+    return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
+}
+export type FunctionResult = ({
+
+} | {
+    error: string;
+}) & {
+    timeTaken: number;
+    evalTime: number;
+    runCount: number;
+};
+
+let currentCallSpec: CallSpec | undefined;
+export function getCurrentCall() {
+    if (!currentCallSpec) {
+        debugbreak(2);
+        debugger;
+        throw new Error("Not presently in a call, and so cannot get call");
+    }
+    return currentCallSpec;
+}
+export function getCurrentCallAllowUndefined() {
+    return currentCallSpec;
+}
+export function overrideCurrentCall<T>(call: CallSpec, code: () => T) {
+    let prev = currentCallSpec;
+    currentCallSpec = call;
+    try {
+        return code();
+    } finally {
+        currentCallSpec = prev;
+    }
+}
+
+function getDebugName(call: CallSpec, functionSpec: FunctionSpec, colored = false) {
+    let pathName = getPathFromStr(functionSpec.exportPathStr).slice(1).join(".");
+    let mainPart = `${call.DomainName}:${functionSpec.FilePath}|${pathName}|${debugTime(call.runAtTime)}`;
+    if (colored) {
+        return `${magenta(mainPart)}`;
+    } else {
+        return `${mainPart}`;
+    }
+}
+
+type PermissionsCheckType = {
+    new(callerMachineId: { callerMachineId: string; callerIP: string; }): {
+        checkPermissions(path: string): { permissionsPath: string; allowed: boolean; };
+    };
+    skipPermissionsChecks: <T>(code: () => T) => T;
+}
+
+export type DebugFunctionShardInfo = {
+    domainName: string;
+    shardRange: { startFraction: number, endFraction: number };
+    secondaryShardRange?: { startFraction: number, endFraction: number };
+};
+let debugFunctionRunnerShards: DebugFunctionShardInfo[] = [];
+export function debug_getFunctionRunnerShards() {
+    return debugFunctionRunnerShards;
+}
+setImmediate(() => {
+    set_debug_getFunctionRunnerShards(debug_getFunctionRunnerShards);
+});
+
+export class PathFunctionRunner {
+    public static RUN_START_COUNT = 0;
+    public static RUN_FINISH_COUNT = 0;
+    public static DEBUG_CALLS = false;
+    public static DEBUG_CALL_TRIGGERS = false;
+    public static DEBUG_WATCHES_THRESHOLD = 5;
+    public static MAX_WATCH_LOOPS = 1000;
+
+    constructor(private config: {
+        domainName: string;
+        shardRange: { startFraction: number, endFraction: number };
+        secondaryShardRange?: { startFraction: number, endFraction: number };
+        PermissionsChecker: PermissionsCheckType | undefined;
+    }) {
+        debugFunctionRunnerShards.push({
+            domainName: config.domainName,
+            shardRange: config.shardRange,
+            secondaryShardRange: config.secondaryShardRange,
+        });
+        logErrors(this.startWatching());
+    }
+
+    // Calls runCall on any calls that match our domain + shard range
+    private async startWatching() {
+        let { shardRange, secondaryShardRange } = this.config;
+
+        // We will use PathValueProxyWatcher to watch the paths
+        // Maintain only one outstanding runCall per callId
+        //  - Once it finishes, we can trigger another call, if we notice the result is not set.
+        //      - AS we should get write prediction for the result, we can rerun our proxy check
+        //          loop immediately.
+
+        let fullFraction = shardRange;
+        if (secondaryShardRange) {
+            fullFraction = {
+                startFraction: Math.min(shardRange.startFraction, secondaryShardRange.startFraction),
+                endFraction: Math.max(shardRange.endFraction, secondaryShardRange.endFraction),
+            };
+        }
+
+        let config = this.config;
+
+        let self = this;
+
+        let outstandingCalls = 0;
+
+        let watchModuleCalls = cache((moduleId: string) => {
+            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls);
+            // Object.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results);
+            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, fullFraction);
+            Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, fullFraction);
+        });
+
+        let callsToCall = new Map<string, { call: CallSpec; functionSpec: FunctionSpec }>();
+        let runningCalls = new Set<string>();
+
+        let callIdsToUnwatch = new Set<{ filePath: string; callId: string; }>();
+
+        // Rarely we might need to queue a function multiple times, when we are late to receive rejections.
+        //  HOWEVER, after enough times, we should stop, as we will probably just infinitely queue it.
+        const MAX_QUEUE_COUNT = 25;
+        let queueLimitCounts = new Map<string, number>();
+        // Clear every hour, so we don't leave
+        runInfinitePoll(timeInHour, () => queueLimitCounts.clear());
+
+        const domainName = config.domainName;
+        const watcher = proxyWatcher.createWatcher({
+            trackTriggers: true,
+            skipPermissionsCheck: true,
+            // Maybe allow running with unsynced reads? This might be good, because we don't need all reads
+            //  to detect calls. BUT, it might also be bad, because it might cause us to rerun in a cascading
+            //  fashion? (As in, be really slow).
+            //  - Shouldn't be needed, because writes don't take that long
+            //  - Could fix bugs where we never get a read, and therefore all function calls fails to run?
+            // allowUnsyncedReads: true
+            watchFunction: function findFunctionsToCall() {
+                // TODO: Maybe abstract the "preserve watchers" stuff,
+                //  so that it is automatically done at the start of our trigger.
+                const watcher = proxyWatcher.getTriggeredWatcher();
+
+                // Preserve previous watches explicitly (so we don't have to use the proxy, which would be slower)
+                {
+                    for (let path of watcher.lastUnsyncedAccesses) {
+                        if (!authorityStorage.isSynced(path)) {
+                            watcher.pendingUnsyncedAccesses.add(path);
+                        }
+                    }
+                    for (let path of watcher.lastUnsyncedParentAccesses) {
+                        if (!authorityStorage.isParentSynced(path)) {
+                            watcher.pendingUnsyncedParentAccesses.add(path);
+                        }
+                    }
+                    // NOTE: If we don't clone, then pathValueClientWatcher cannot detect changes, and we
+                    //  end up staying watched on everything!
+                    watcher.pendingWatches.paths = new Set(watcher.lastWatches.paths);
+                    watcher.pendingWatches.parentPaths = new Set(watcher.lastWatches.parentPaths);
+                }
+
+
+
+                let data = () => functionSchema()[domainName].PathFunctionRunner;
+
+                for (let { filePath, callId } of callIdsToUnwatch) {
+                    watcher.pendingWatches.paths.delete(getProxyPath(() => data()[filePath].Calls[callId]));
+                    watcher.pendingWatches.paths.delete(getProxyPath(() => data()[filePath].Results[callId]));
+                }
+                callIdsToUnwatch.clear();
+                for (let [filePath, moduleData] of Object.entries(data())) {
+                    watchModuleCalls(filePath);
+                }
+
+                let triggerPaths = new Set<string>();
+                for (let [filePath, moduleData] of Object.entries(data())) {
+                    let callPath = getProxyPath(() => data()[filePath].Calls);
+                    triggerPaths.add(callPath);
+
+                    // ALSO, Result changes (result rejections), can trigger a run
+                    let resultPath = getProxyPath(() => data()[filePath].Results);
+                    triggerPaths.add(resultPath);
+                }
+
+                let callPathLength = 0;
+                if (triggerPaths.size > 0) {
+                    callPathLength = getPathDepth(Array.from(triggerPaths)[0]);
+                }
+
+                let newCalls: { call: CallSpec; functionSpec: FunctionSpec }[] = [];
+
+                if (watcher.triggeredByChanges) {
+                    for (let path of watcher.triggeredByChanges.paths) {
+                        let callPathEquivalent = trimPathStrToDepth(path, callPathLength);
+                        if (triggerPaths.has(callPathEquivalent)) {
+                            const callId = getPathIndex(path, callPathLength);
+                            if (!callId) continue;
+                            let moduleId = getPathIndex(path, callPathLength - 2);
+                            if (!moduleId) continue;
+
+                            callsToCall.delete(callId);
+
+                            let moduleData = data()[moduleId];
+
+                            let result = atomicObjectRead(moduleData.Results[callId]);
+                            if (result) continue;
+
+                            let callData = atomicObjectRead(moduleData.Calls[callId]);
+                            if (!callData) continue;
+
+                            if (callData.DomainName !== domainName) continue;
+                            if (callData.ModuleId !== moduleId) continue;
+                            if (callData.CallId !== callId) continue;
+
+                            let functionSpec = atomicObjectRead(moduleData.Sources[callData.FunctionId]);
+                            if (!functionSpec) {
+                                if (isSynced(moduleData.Sources[callData.FunctionId])) {
+                                    console.warn(yellow(`Cannot call ${callData.DomainName}.${callData.ModuleId}.Sources.${callData.FunctionId} because it is not deployed`));
+                                }
+                                continue;
+                            }
+
+                            // If we haven't synced the result, we can't know if it is ready or not
+                            //  (we'll run anyways once it is synced). Most other parts are immutable,
+                            //  but the result is by definition not, so... it is one of the values we check!
+                            if (!isSynced(moduleData.Results[callId])) continue;
+
+                            if (runningCalls.has(callId)) continue;
+                            runningCalls.add(callId);
+
+                            let limitCount = queueLimitCounts.get(callId) || 0;
+                            limitCount++;
+                            queueLimitCounts.set(callId, limitCount);
+                            // NOTE: Calls are event writes, so... they should just clean themselves up, after we ignore them for long enough.
+                            if (limitCount >= MAX_QUEUE_COUNT) {
+                                // Only error the first time, as we don't need need that many errors
+                                if (limitCount === MAX_QUEUE_COUNT) {
+                                    console.error(`Queue limit reached (${limitCount}) for ${getDebugName(callData, functionSpec, true)}`);
+                                }
+                                continue;
+                            }
+
+                            if (PathFunctionRunner.DEBUG_CALLS) {
+                                console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
+                                let resultsPath = getProxyPath(() => moduleData.Results[callId]);
+                                let history = authorityStorage.getValuePlusHistory(resultsPath);
+                                console.log(`    History: ${history.length}`);
+                                for (let { valid, time, canGCValue } of history) {
+                                    console.log(`        ${valid ? "✅" : "❌"} ${debugTime(time)} ${canGCValue ? "value is undefined" : ""}`);
+                                }
+                            }
+                            newCalls.push({ call: callData, functionSpec });
+                        }
+                    }
+                }
+
+                // Better run first calls first, or else we will just pointlessly cause rejections.
+                newCalls.sort((a, b) => compareTime(a.call.runAtTime, b.call.runAtTime));
+                for (let { call, functionSpec } of newCalls) {
+                    // We don't need wait for all pending synced accesses to finish.
+                    //  Worst case we have an old functionSpec, and then runCall has to loop again.
+                    //  Otherwise the call request is (or should be) atomic, so it this should just work...
+                    //  - Or if it is rejected, we will detect it has no result, and run it again.
+                    // NOTE: We run all calls in parallel, so we can load data in parallel. This is a bit dangerous, and might
+                    //      result in out of order execution, but... we commit at runTimes, so, this should eventually
+                    //      resolve itself due to rejections (which will change the call result, which SHOULD cause us to rerun again).
+                    //      - This is a bit dangerous, as we might overwhelm ourself. We should address this one we support
+                    //          FunctionRunner sharding.
+                    outstandingCalls++;
+                    let runCallPromise = self.runCall(call, functionSpec);
+                    void runCallPromise.finally(() => {
+                        outstandingCalls--;
+                        runningCalls.delete(call.CallId);
+                    });
+                    logErrors(runCallPromise);
+                }
+            },
+        });
+        registerDynamicResource("paths|PathFunctionRunner.watches", () => watcher.lastWatches.paths.size);
+        registerDynamicResource("paths|PathFunctionRunner.outstandingCalls", () => outstandingCalls);
+    }
+
+
+    @measureFnc
+    private async getExportsFromSpec(spec: FunctionSpec): Promise<unknown> {
+        let module = await getModuleFromSpec(spec);
+        return module.exports;
+    }
+
+    private getSecondaryShardDelay(callPath: CallSpec): number {
+        let { shardRange, secondaryShardRange } = this.config;
+        if (!secondaryShardRange) return 0;
+        let fraction = __getRoutingHash(callPath.CallId);
+        // It isn't secondary if it is primary
+        if (shardRange.startFraction <= fraction && fraction < shardRange.endFraction) return 0;
+        if (!(secondaryShardRange.startFraction <= fraction && fraction < secondaryShardRange.endFraction)) return 0;
+        let fractionDistanceToPrimary = Math.min(
+            Math.abs(fraction - shardRange.startFraction),
+            Math.abs(fraction - shardRange.endFraction),
+        );
+        return Math.min(
+            // No more than 50% of accepted change, otherwise it might not be accepted
+            MAX_ACCEPTED_CHANGE_AGE * 0.5,
+            // Default values in case MAX_ACCEPTED_CHANGE_AGE is VERY high (minutes, or even hours)
+            30_000 + 10_000 * fractionDistanceToPrimary,
+            // Wait 5000 seconds, then start 
+            5000 + MAX_ACCEPTED_CHANGE_AGE * fractionDistanceToPrimary * 0.05 / (shardRange.endFraction - shardRange.startFraction),
+        );
+    };
+
+    private callsRemainingInTick = cache((wait: true) => {
+        // NOTE: This gives us 100K calls per second, which is more than we can run anyways.
+        //  We just need to wait a bit, otherwise the IO loop can never run, and so our finished calls
+        //  can never actually finish, until eventually they are too late to finish and get rejected!
+        let tickDone = delay(10);
+        void tickDone.finally(() => {
+            this.callsRemainingInTick.clear(true);
+        });
+        return { calls: 1000, tickDone };
+    });
+    private async letIOClear(): Promise<void> {
+        while (true) {
+            let callsObj = this.callsRemainingInTick(true);
+            callsObj.calls--;
+            if (callsObj.calls >= 0) return;
+            await callsObj.tickDone;
+        }
+    }
+
+    private async runCall(callPath: CallSpec, functionSpec: FunctionSpec): Promise<void> {
+        const PermissionsChecker = this.config.PermissionsChecker;
+        let skipPermissions = <T>(code: () => T) => code();
+        if (PermissionsChecker) {
+            skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
+        }
+
+        await this.letIOClear();
+
+        let secondaryDelay = this.getSecondaryShardDelay(callPath);
+        if (secondaryDelay) {
+            async function getIsFinished() {
+                return await proxyWatcher.commitFunction({
+                    watchFunction() {
+                        let syncedModule = skipPermissions(() =>
+                            functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId]
+                        );
+                        return !!atomicObjectRead(syncedModule.Results[callPath.CallId]);
+                    },
+                });
+            }
+            // Call once, so we are already watching the 
+            await delay(secondaryDelay);
+            let isFinished = await getIsFinished();
+            if (isFinished) return;
+            if (PathFunctionRunner.DEBUG_CALLS) {
+                let fraction = __getRoutingHash(callPath.CallId);
+                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${getDebugName(callPath, functionSpec, true)}`);
+            }
+        }
+
+        PathFunctionRunner.RUN_START_COUNT++;
+        if (PathFunctionRunner.DEBUG_CALLS) {
+            console.log(`STARTING ${getDebugName(callPath, functionSpec, true)}`);
+        }
+        let startTime = Date.now();
+        let runCount = 0;
+
+        let finalWrites: PathValue[] | undefined;
+
+        let evalTime = 0;
+        let retries = 0;
+
+        let nooped = false;
+
+        while (true) {
+            let isFunctionSpecOutdated = false;
+            try {
+
+                let moduleExports = await this.getExportsFromSpec(functionSpec);
+
+                // ALWAYS get the permissions as well. This isn't perfect, as we could very well access data in other files, but... it helps
+                //  reduce extra iterations inside of commitFunction...
+                await getModuleFromSpec({ ...functionSpec, exportPathStr: getExportPath(PERMISSIONS_FUNCTION_ID), FunctionId: PERMISSIONS_FUNCTION_ID, });
+
+                let exportPath = getPathFromStr(functionSpec.exportPathStr);
+                let exportObj = moduleExports as any;
+                for (let path of exportPath) {
+                    exportObj = exportObj[path];
+                }
+                let baseFunction = exportObj as Function;
+
+                if (typeof baseFunction !== "function") {
+                    throw new Error(`Export at ${JSON.stringify(exportPath.join("."))} was not a function (it was ${typeof baseFunction})`);
+                }
+
+                await proxyWatcher.commitFunction({
+                    canWrite: true,
+                    debugName: getDebugName(callPath, functionSpec),
+                    runAtTime: callPath.runAtTime,
+                    getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callPath)),
+                    nestedCalls: "inline",
+                    watchFunction: function runCallWatcher() {
+                        runCount++;
+                        if (PathFunctionRunner.DEBUG_CALLS) {
+                            console.log(`Evaluating (try count ${runCount}) ${getDebugName(callPath, functionSpec, true)}`);
+                            console.log(`    RUNNING AT REAL TIME ${debugTime(proxyWatcher.getTriggeredWatcher().currentReadTime || callPath.runAtTime)}`);
+                        }
+                        if (PathFunctionRunner.DEBUG_CALL_TRIGGERS && runCount > PathFunctionRunner.DEBUG_WATCHES_THRESHOLD) {
+                            // NOTE: If this happens a few times during initial loading it is fine. If it happens a lot... and with values MUCH
+                            //  greater than 2... it is likely a problem, and your functions might be doing cascading loading, which can take
+                            //  a very long time (ex, `for(let i = 0; i < 1000; i++) if (!arr[i]) return`)
+                            console.log(`> 2 call evaluations (try count ${runCount}) ${getDebugName(callPath, functionSpec, true)}`);
+                            console.log(yellow(`    Paths changed`));
+                            for (let path of proxyWatcher.getTriggeredWatcher().triggeredByChanges?.paths || []) {
+                                console.log(yellow(`        ${path}`));
+                            }
+                        }
+                        if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
+                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}`);
+                        }
+
+                        // We NEED to depend on the function spec, so spec updates can be atomic (which is needed to deploy an update
+                        //  to many nodes at once, without having any period where multiple versions both commit values).
+                        let syncedModule = skipPermissions(() =>
+                            functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId]
+                        );
+                        let syncedSpec = skipPermissions(() =>
+                            atomicObjectRead(syncedModule.Sources[callPath.FunctionId])
+                        );
+
+                        //  (We also need to depend on the RIGHT function spec).
+                        if (
+                            syncedSpec && (
+                                syncedSpec.DomainName !== functionSpec.DomainName
+                                || syncedSpec.ModuleId !== functionSpec.ModuleId
+                                || syncedSpec.FilePath !== functionSpec.FilePath
+                                || syncedSpec.FunctionId !== functionSpec.FunctionId
+                                || syncedSpec.exportPathStr !== functionSpec.exportPathStr
+                                || syncedSpec.FilePath !== functionSpec.FilePath
+                                || syncedSpec.gitRef !== functionSpec.gitRef
+                                || syncedSpec.gitURL !== functionSpec.gitURL
+                            )
+                        ) {
+                            isFunctionSpecOutdated = true;
+                            functionSpec = {
+                                DomainName: syncedSpec.DomainName,
+                                ModuleId: syncedSpec.ModuleId,
+                                FilePath: syncedSpec.FilePath,
+                                FunctionId: syncedSpec.FunctionId,
+                                exportPathStr: syncedSpec.exportPathStr,
+                                gitRef: syncedSpec.gitRef,
+                                gitURL: syncedSpec.gitURL,
+                            };
+                            return;
+                        }
+
+                        let evalTimeStart = Date.now();
+                        let args = parseArgs(callPath);
+                        overrideCurrentCall(callPath, () => {
+                            baseFunction(...args);
+                        });
+                        evalTime += Date.now() - evalTimeStart;
+
+                        // NOTE: The results are only temporarily stored, but serve as a way to let clients
+                        //  know figure out the dependencies for the call, which makes function call prediction
+                        //  possible.
+                        skipPermissions(() => {
+                            let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callPath.CallId]));
+                            if (result) {
+                                nooped = true;
+                                // It is important to NOOP if it has already been written. Otherwise there could be multiple
+                                //  runs of the same function, which could easily vary (if they use Math.random(), etc).
+                                //  WHICH, as the write time is predetermined, would cause ValuePaths with equal times, but
+                                //  different values (which breaks a lot of assumptions much of the code depends on! such
+                                //  as locks, etc).
+                                if (!secondaryDelay && Querysub.isAllSynced()) {
+                                    console.warn(`Skipping function write, as it has already been written to by another FunctionRunner.`);
+                                }
+                                return;
+                            }
+                            nooped = false;
+                            syncedModule.Results[callPath.CallId] = atomicObjectWrite({
+                                runCount: runCount,
+                                timeTaken: Date.now() - evalTimeStart,
+                                evalTime,
+                            });
+                            proxyWatcher.setEventPath(() => syncedModule.Results[callPath.CallId]);
+                        });
+                    },
+                }, {
+                    onWritesCommitted(writes) {
+                        finalWrites = writes;
+                    },
+                });
+            } catch (e: any) {
+                if (isFunctionSpecOutdated) continue;
+
+                // NOTE: We are changing our paradigm to reject early, rather than retry. This is more stable. You can't really commit
+                //  important values and then just leave, unless those values have no locks. This helps prevent values from being
+                //  in different states on different machines, by just rejecting the call.
+                // if (e.message.includes("MAX_CHANGE_AGE_EXCEEDED") && retries < 10 && evalTime < MAX_ACCEPTED_CHANGE_AGE) {
+                //     // NOTE: This should never really happen. It is only really if our PathValueServer goes down for a minute or so,
+                //     //  so we don't just drop all of our function calls.
+                //     //  - This is why we log every time, because no user actions should be able to result in this happening frequently.
+                //     console.error(red(`RETRYING due to MAX_CHANGE_AGE_EXCEEDED. Hopefully the server will be more responsive this time (retry ${retries}): ${getDebugName(callPath, functionSpec, true)}`));
+                //     // NOTE: Updating the time is fine, and won't break prediction rejections, as we reject based on path, not time.
+                //     //  (It might result in the prediction being wrong, but it is better than the user action being completely removed).
+                //     callPath = { ...callPath, runAtTime: getNextTime() };
+                //     retries++;
+                //     continue;
+                // }
+                //if (PathFunctionRunner.DEBUG_CALLS)
+                {
+                    console.log(`ERROR ${getDebugName(callPath, functionSpec, true)}`, e.stack);
+                }
+
+                // NOTE: commitFunction should never throw? If the server is dead, it still won't throw, so... we aren't try/catching it,
+                //  as I don't see how retrying would help (the same error would almost certainly happen again and again).
+                await proxyWatcher.commitFunction({
+                    canWrite: true,
+                    // NOTE: We don't set runAtTime, because the rejection might be due to the call being too old
+                    debugName: `error (${getDebugName(callPath, functionSpec)})`,
+                    eventWrite: true,
+                    doNotStoreWritesAsPredictions: true,
+                    watchFunction() {
+                        let syncedModule = functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId];
+                        syncedModule.Results[callPath.CallId] = atomicObjectWrite({
+                            error: e.stack || "",
+                            runCount: runCount,
+                            timeTaken: Date.now() - startTime,
+                            evalTime,
+                        });
+                    },
+                }, {
+                    onWritesCommitted(writes) {
+                        finalWrites = writes;
+                    },
+                });
+            }
+            if (isFunctionSpecOutdated) continue;
+            break;
+        }
+
+        if (PathFunctionRunner.DEBUG_CALLS) {
+            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callPath, functionSpec, true)}, writes: ${finalWrites?.length}`);
+        }
+        PathFunctionRunner.RUN_FINISH_COUNT++;
+    }
+}