querysub 0.433.0 → 0.437.0

package/src/4-querysub/querysubPrediction.ts

@@ -1,484 +1,484 @@
-import { cacheJSONArgsEqual, lazy } from "socket-function/src/caching";
-import { MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, predictionLockVersion } from "../0-path-value-core/pathValueCore";
-import { validStateComputer } from "../0-path-value-core/ValidStateComputer";
-import { proxyWatcher, atomicObjectRead, DryRunResult, getCurrentCallCreationProxy } from "../2-proxy/PathValueProxyWatcher";
-import { getProxyPath } from "../2-proxy/pathValueProxy";
-import { CallSpec, FunctionResult, FunctionSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
-import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
-import { logErrors } from "../errors";
-import { getPathFromStr, getPathStr1 } from "../path";
-import { Querysub, QuerysubController, QuerysubControllerBase, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
-import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
-import { magenta, red } from "socket-function/src/formatting/logColors";
-import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-
-import { setFlag } from "socket-function/require/compileFlags";
-import cbor from "cbor-x";
-import { FunctionMetadata } from "../3-path-functions/syncSchema";
-import { PromiseObj, isNode } from "socket-function/src/misc";
-import { isPublic } from "../config";
-import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
-import { addEpsilons } from "../bits";
-import { delay } from "socket-function/src/batching";
-setFlag(require, "cbor-x", "allowclient", true);
-const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
-
-let onPredictionFinishedCallbacks: Array<(data: { callId: string; result: FunctionResult; functionId: string }) => void> = [];
-
-export function onPredictionFinished(callback: (data: {
-    callId: string;
-    result: FunctionResult;
-    functionId: string;
-}) => void) {
-    onPredictionFinishedCallbacks.push(callback);
-}
-
-// NOTE: Most functions won't use this, as they should use regular api calls. However,
-//  as we are using paths for RequireJS, we explicitly need our local (when serverside).
-async function getPredictController() {
-    let controller: QuerysubControllerBase;
-    if (isNode()) {
-        // NOTE: If we are on node, then the require WON'T run over the network, so we need to use
-        //  our local module path, not the remote one.
-        controller = new QuerysubControllerBase();
-    } else {
-        let nodeId = await querysubNodeId();
-        if (!nodeId) throw new Error("No querysub node found");
-        controller = QuerysubController.nodes[nodeId] as any;
-    }
-    return controller;
-}
-// TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
-//  but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
-//  do too much for us if we already have the fully resolved path...
-//  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
-const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<true> => {
-    let controller = await getPredictController();
-
-    let path = await controller.getModulePath({ functionSpec: spec });
-    setGitURLMapping({ spec, resolvedPath: path });
-    return true;
-});
-
-const getDevFunctionSpecFromCall = cacheJSONArgsEqual(async (call: {
-    DomainName: string;
-    ModuleId: string;
-    FunctionId: string;
-}) => {
-    let controller = await getPredictController();
-    return controller.getDevFunctionSpecFromCall(call);
-});
-
-
-export function getCallResultPath(call: CallSpec) {
-    return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
-}
-export function getCallResult(call: CallSpec) {
-    return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId];
-}
-
-/** Force predictions to run in the trigger order, so they can resolve an be added before
- *      the next predictions. Also, to preserve call order.
- */
-//const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult | undefined>) => run());
-// UPDATE: We no longer run them in serial, as this caused a lot of lag if we ran a lot of functions which access unique values they needed to sync. Generally speaking, most predictions should run in serial anyway, As most functions should be accessing already synchronized values. 
-const predictRunCommitLoop = (run: () => Promise<PredictResult | undefined>) => run();
-
-
-
-
-// IMPORTANT! This has to be synchronous, and we need to synchronously get into the proxy watcher call. That way other calls after this know to wait for our proxy watcher to finish. Otherwise, it's very, very, very easy to write code, where you call a function, you expect it to write to a value, and then you run some other code, maybe which just uses a commit async, and tries to read from that value. But the value won't exist, because we won't even have started the prediction yet, and the proxy watcher can't possibly know to wait, because we haven't started the prediction. So... this has to be synchronous!
-export function predictCall(call: CallSpec, metadata: FunctionMetadata): {
-    onApplied: () => void;
-    cancel: () => void;
-    predictPromise: Promise<PredictResult | undefined>;
-} {
-    let predictObj = predictCallBase({ call, metadata });
-    let cancel = predictObj.cancel;
-    let onApplied = predictObj.onApplied;
-    registerPredictionBlocker(call.CallId, predictObj.predictPromise);
-    return { cancel, onApplied, predictPromise: predictObj.predictPromise };
-}
-
-function getPredictTime(time: Time) {
-    return { time: time.time, version: predictionLockVersion, creatorId: time.creatorId };
-}
-
-export interface PredictResult {
-    readPaths: Set<string>;
-    readParentPaths: Set<string>;
-    writes: PathValue[];
-    /** Writes that have been replaced AT the write time of the prediction */
-    replacedWriteValues: PathValue[];
-}
-function predictCallBase(config: {
-    call: CallSpec;
-    metadata: FunctionMetadata;
-    overrides?: PathValue[];
-}): {
-    onApplied: () => void;
-    cancel: () => void;
-    predictPromise: Promise<PredictResult | undefined>;
-} {
-    let call = config.call;
-    let pathResultWrite = getCallResultPath(call);
-
-    let actualValueFinished = new PromiseObj();
-    function onActualFinished() {
-        if (actualValueFinished.resolveCalled) return;
-        let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
-        let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
-        if (!result) return;
-        if (result.lastInternalLoopCount === -1) return;
-        cleanupPrediction();
-        actualValueFinished.resolve();
-        for (let callback of onPredictionFinishedCallbacks) {
-            callback({ callId: call.CallId, result, functionId: call.FunctionId });
-        }
-    }
-    function onApplied() {
-        clientWatcher.setWatches({
-            callback: onActualFinished,
-            paths: new Set([pathResultWrite]),
-            parentPaths: new Set(),
-        });
-    }
-
-    let functionResult: FunctionResult = {
-        // Special prediction values (we don't even really need to put anything here, but it
-        //  might as well have the right format, in case a bug causes it to be used somewhere)
-        lastInternalLoopCount: -1,
-        outerLoopCount: -1,
-        totalInternalLoopCount: -1,
-        timeTaken: -1,
-        totalTime: -1,
-        evalTime: 0,
-    };
-
-    // NOTE: We don't cascade predictions, instead only depending on the call succeeding. This is because if we depend on writes,
-    //  we will be invalidated before we get the actual call result, which results in going back in time, a delay, then
-    //  having the actual call results, and so jumping forward again. It is better to just wait, so our latest value
-    //  becomes the latest call value, when we receive it.
-    let predictResultWrite: PathValue = {
-        path: pathResultWrite,
-        value: functionResult,
-        locks: [
-            {
-                // We use a very low negative version, so we don't have to worry about this actually existing. We don't really want
-                //  a lock on startTime, we just want the range lock.
-                startTime: { time: call.runAtTime.time, version: -2344282313, creatorId: call.runAtTime.creatorId },
-                // NOTE: The version is special, and used to detect prediction rejections in some logging.
-                // NOTE: We need to set our end time enough in the future so errors cause us to be rejected
-                endTime: { time: call.runAtTime.time + MAX_CHANGE_AGE, version: Number.MAX_SAFE_INTEGER, creatorId: call.runAtTime.creatorId },
-                path: pathResultWrite,
-                // Technically we are depending on a future time, so... we are depending on ourself? And our value isn't undefined.
-                readIsTransparent: false,
-            }
-        ],
-        lockCount: 1,
-        // We right our result BEFORE the actual result, so it will be clobbered (and before our read lock!)
-        //  when the result arrives (although... it will ALSO be rejected, so... this might be less important...)
-        //  (Also, more importantly, this ha to be UNIQUE, otherwise time locks break!)
-        time: getPredictTime(call.runAtTime),
-        valid: true,
-        event: true,
-    };
-    let predictLocks: ReadLock[] = [
-        {
-            startTime: predictResultWrite.time,
-            endTime: predictResultWrite.time,
-            path: pathResultWrite,
-            readIsTransparent: false,
-        }
-    ];
-    let predictions: PredictResult | undefined;
-    let predictPromise = predictRunCommitLoop(async () => {
-        // IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
-        if (Querysub.DEBUG_PREDICTIONS) {
-            console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
-        }
-
-        let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
-
-        let dryRunResult: {
-            writes: PathValue[];
-            readPaths: Set<string>;
-            readParentPaths: Set<string>;
-        };
-
-        setTimeout(() => {
-            clientWatcher.unwatch(onActualFinished);
-        }, 30 * 1000);
-
-        try {
-            let tempDryRunResult = await Promise.race([
-                await getCallWrites({ call, debugName, overrides: config.overrides, useFinishReordering: true, metadata: config.metadata }),
-                actualValueFinished.promise,
-            ]);
-            if (!tempDryRunResult) {
-                if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
-                    console.log(magenta(`Abort predict call before prediction finished, already received call result`), `${call.DomainName}.${call.FunctionId}`);
-                }
-                return undefined;
-            }
-            dryRunResult = tempDryRunResult;
-        } catch (e: any) {
-            if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
-                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
-            } else {
-                // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
-                //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
-                //  the race to predict the function against the server updating it, it is likely our prediction will now
-                //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
-                //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
-            }
-            return {
-                writes: [],
-                readPaths: new Set(),
-                readParentPaths: new Set(),
-                replacedWriteValues: [],
-            };
-        }
-        predictions = {
-            writes: dryRunResult.writes,
-            readPaths: dryRunResult.readPaths,
-            readParentPaths: dryRunResult.readParentPaths,
-            replacedWriteValues: dryRunResult.writes.map(write => {
-                let path = write.path;
-                let newWrite: PathValue = authorityStorage.getValueAtTime(write.path, write.time) || {
-                    path, valid: true, time: write.time, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true,
-                };
-                newWrite = { ...newWrite };
-                // Use a time very slightly after, so it clobbers the write, and any writes before it, but none after.
-                newWrite.time = write.time;
-                write.time = {
-                    time: write.time.time,
-                    version: write.time.version,
-                    creatorId: write.time.creatorId + 1,
-                };
-                return newWrite;
-            }),
-        };
-
-        // Abort, and don't predict
-        if (didCancel) return predictions;
-
-        for (let value of predictions.writes) {
-            value.source = debugName;
-        }
-
-        // If we already received the actual call, we can't add the prediction, as... the invalidation
-        //      code won't properly immediately reject our prediction, as we are not the authority on the
-        //      path, so it treats it as source of truth.
-        if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
-            if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
-                console.log(magenta(`Abort predict call, already received call result`), `${call.DomainName}.${call.FunctionId}`);
-            }
-            return predictions;
-        }
-
-        // We commit slightly different writes, for  lock invalidation. Our caller doesn't need / doesn't want this
-        //  extra write, so we just don't return it.
-        // NOTE: I guess we technically don't need to actually write to the result, but... might as well?
-        predictions.writes.push(predictResultWrite);
-        // Update all locks to use the same locks as predictResultWrite
-        // Ensure we use the predict time, otherwise these times will be reused when the call actually happens,
-        //  potentially with different values, which breaks all of our ReadLocks!
-        for (let predict of predictions.writes) {
-            predict.time = predictResultWrite.time;
-            predict.locks = predictLocks;
-            predict.lockCount = predictLocks.length;
-        }
-
-        validStateComputer.ingestValuesAndValidStates({
-            pathValues: predictions.writes,
-            parentSyncs: [],
-            initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: true,
-        });
-
-        if (Querysub.DEBUG_PREDICTIONS) {
-            console.log(magenta(`Finished and applied predict call`), `${call.DomainName}.${call.FunctionId}`);
-        }
-        return predictions;
-    });
-    logErrors(predictPromise);
-
-    let didCancel = false;
-    function rejectPrediction() {
-        if (didCancel) return;
-        didCancel = true;
-        if (!predictions) return;
-        // Reject our predictions, as the call likely never got committed, so it will never be written
-        validStateComputer.ingestValuesAndValidStates({
-            pathValues: predictions.writes.map(write => ({
-                path: write.path,
-                value: write.value,
-                locks: [],
-                lockCount: 0,
-                valid: false,
-                time: predictResultWrite.time,
-                isTransparent: false,
-            })),
-            parentSyncs: [],
-            initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: true,
-        });
-    }
-
-    setTimeout(cleanupPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
-    function cleanupPrediction() {
-        if (didCancel) return;
-
-        // ALWAYS reject the prediction, in case the function runner server is down.
-        //  - ALSO, once it goes back up, the write likely will be too far in the future, so our lock won't be rejected,
-        //      will will cause the bad value to stick around.
-        //      We could fix this with an endTime infinitely in the future, but then the lock can never be GCed,
-        //      which create a memory leak.
-        rejectPrediction();
-
-        if (Querysub.AUDIT_PREDICTIONS && predictions) {
-            const afterTime = { time: call.runAtTime.time, version: Number.MAX_SAFE_INTEGER, creatorId: 0 };
-            // Clone predictions, to strip symbols
-            let predictionsCopy = cborEncoder().decode(cborEncoder().encode(predictions)) as typeof predictions;
-            for (let predict of predictionsCopy.writes ?? []) {
-                if (predict.path === pathResultWrite) continue;
-                let finalValueObj = authorityStorage.getValueAtTime(predict.path, afterTime);
-                if (!authorityStorage.isSynced(predict.path)) continue;
-
-                let finalValue = pathValueSerializer.getPathValue(finalValueObj);
-                let predictValue = pathValueSerializer.getPathValue(predict);
-
-                let finalBuffer = cborEncoder().encode(finalValue);
-                let predictBuffer = cborEncoder().encode(predictValue);
-                // If they are different, warn
-                if (!finalBuffer.equals(predictBuffer)) {
-                    // NOTE: This MIGHT be due to delayCommit on another function. We can't know when to force flush
-                    //  delayCommit values, and so we only force flush if other (non-delayCommit) functions are committed
-                    //  in the same schema, which will be wrong if data is accessed cross schema.
-                    //  - If this happens, and is a problem, we COULD handle it, but... it is very difficult, as we would need
-                    //      to evaluate the other function, detect it conflicts with a delayCommit functions, then run the delayCommit
-                    //      function, then re-evaluated the conflicted prediction (which is a lot of communication and reruns
-                    //      between a lot of systems... especially because in a second this will resolve by the server
-                    //      clobbering our prediction).
-                    //      - AND, we can't commit the pending calls until we sort out the order, so this necessarily requires
-                    //          slowing down commits if we are delaying other calls.
-                    console.warn(`${red("Prediction was wrong")}: for ${call.DomainName}.${call.FunctionId} value path ${getPathFromStr(predict.path).join(".")} predict != finalValue. ${config.metadata.delayCommit && "This function is using delay commit. It's likely that you are using the time from Querysub.now() (or accessing the function call time in some other way). This doesn't work with delay commit because the server will rewrite the commit time, instead you should pass the timestamp as a parameter (And then make sure it's not some kind of privilege value that the user could cheat)" || ""} It might be the case that you shouldn't even predict the call client side (you can use the functionMetadata to set it as nopredict). It might also be the case that you do want to predict it, but you should batch the calls, so you only make one call, instead of many calls at once.`, predictValue, finalValue);
-                }
-            }
-        }
-    }
-
-    return {
-        cancel: rejectPrediction,
-        predictPromise,
-        onApplied,
-    };
-}
-
-const getDevFunctionCache = cacheAsyncLimitedJSON(100_000, getDevFunctionSpecFromCall);
-const addToModuleLoaderCache = cacheAsyncLimitedJSON(100_000, addModuleToLoader);
-
-function getFunctionSpec(call: CallSpec): FunctionSpec | undefined {
-    if (!isPublic()) {
-        let obj = getDevFunctionCache({
-            DomainName: call.DomainName,
-            ModuleId: call.ModuleId,
-            FunctionId: call.FunctionId,
-        });
-        if (!obj) throw new Error(`Function not referenced in deploy.ts ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
-        setGitURLMapping({ spec: obj.functionSpec, resolvedPath: obj.modulePath });
-        return obj.functionSpec;
-    }
-
-    let domainObject = functionSchema()[call.DomainName];
-    let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
-    let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
-    if (!functionSpec) {
-        if (!Querysub.isAllSynced()) return undefined;
-        throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
-    }
-    // Add the module to the loader via asking the server the exact url for this call. The loader will
-    //  then load the code from that url when it ends up running it.
-    if (!addToModuleLoaderCache(functionSpec)) return undefined;
-    return functionSpec;
-}
-
-function getFunctionInfoBase(call: CallSpec): {
-    functionSpec: FunctionSpec;
-    baseFunction: Function;
-} | undefined {
-    let functionSpec = getFunctionSpec(call);
-    if (!functionSpec) return undefined;
-
-    let module = getModuleFromConfig(functionSpec);
-    if (module instanceof Promise) {
-        proxyWatcher.triggerOnPromiseFinish(module, { waitReason: `Loading function ${call.DomainName}.${call.ModuleId}.${call.FunctionId}` });
-        return undefined;
-    }
-    let exportPath = getPathFromStr(functionSpec.exportPathStr);
-    let exportObj = module.exports;
-    for (let path of exportPath) {
-        if (!(path in exportObj)) {
-            throw new Error(`Export not found for call prediction: ${path}, in ${call.DomainName}.${call.ModuleId}.${call.FunctionId}. Have ${Object.keys(exportObj)}`);
-        }
-        exportObj = exportObj[path];
-    }
-    let baseFunction = exportObj as Function;
-
-    return { functionSpec, baseFunction };
-}
-
-// IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
-export function getCallWrites(config: {
-    debugName: string;
-    call: CallSpec;
-    overrides?: PathValue[];
-    useFinishReordering?: boolean;
-    metadata?: FunctionMetadata;
-}): Promise<DryRunResult> {
-    let { call, debugName } = config;
-
-    let finishInStartOrder: number | boolean | undefined;
-
-    if (config.useFinishReordering && !config.metadata?.noFinishInStartOrder && call.fromProxy) {
-        let triggerCaller = getCurrentCallCreationProxy();
-        if (!triggerCaller) {
-            require("debugbreak")(2);
-            debugger;
-            throw new Error(`getCallWrites did not happen synchronously when triggering a call. It's very important that we synchronously start the proxy watcher, so we can make sure that predictions happen in a consistent order. FIX THIS! Find the async weighting that was added and move it inside the watch function using cacheAsync to make it watched. For: ${debugName}`);
-        }
-        finishInStartOrder = addEpsilons(triggerCaller.createTime, 1);
-    }
-    let first = true;
-    return proxyWatcher.dryRunFull({
-        debugName,
-        runAtTime: call.runAtTime,
-        overrideAllowLockDomainsPrefixes: [getPathStr1(call.DomainName)],
-        // We are going to clobber locks anyways, so there is no reason to create them
-        unsafeNoLocks: true,
-        overrides: config.overrides,
-        nestedCalls: "inline",
-        // Run after our trigger function. This will insert us before functions which may have been run before us, but after our trigger function. 
-        finishInStartOrder,
-        predictMetadata: config.metadata,
-        watchFunction() {
-            let fncObj = getFunctionInfoBase(call);
-            if (!fncObj) return undefined;
-            let { functionSpec, baseFunction } = fncObj;
-            return overrideCurrentCall({ spec: call, fnc: functionSpec }, () => {
-                if (first) {
-                    if (Querysub.DEBUG_PREDICTIONS) {
-                        console.log(magenta(`Loaded predict function and first run`), `${call.DomainName}.${call.FunctionId}`);
-                    }
-                }
-                first = false;
-
-                let args = parseArgs(call);
-                let result = baseFunction(...args);
-                return result;
-            });
-        },
-    });
+import { cacheJSONArgsEqual, lazy } from "socket-function/src/caching";
+import { MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, predictionLockVersion } from "../0-path-value-core/pathValueCore";
+import { validStateComputer } from "../0-path-value-core/ValidStateComputer";
+import { proxyWatcher, atomicObjectRead, DryRunResult, getCurrentCallCreationProxy } from "../2-proxy/PathValueProxyWatcher";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+import { CallSpec, FunctionResult, FunctionSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
+import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
+import { logErrors } from "../errors";
+import { getPathFromStr, getPathStr1 } from "../path";
+import { Querysub, QuerysubController, QuerysubControllerBase, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
+import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
+import { magenta, red } from "socket-function/src/formatting/logColors";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+
+import { setFlag } from "socket-function/require/compileFlags";
+import cbor from "cbor-x";
+import { FunctionMetadata } from "../3-path-functions/syncSchema";
+import { PromiseObj, isNode } from "socket-function/src/misc";
+import { isPublic } from "../config";
+import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
+import { addEpsilons } from "../bits";
+import { delay } from "socket-function/src/batching";
+setFlag(require, "cbor-x", "allowclient", true);
+const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
+
+let onPredictionFinishedCallbacks: Array<(data: { callId: string; result: FunctionResult; functionId: string }) => void> = [];
+
+export function onPredictionFinished(callback: (data: {
+    callId: string;
+    result: FunctionResult;
+    functionId: string;
+}) => void) {
+    onPredictionFinishedCallbacks.push(callback);
+}
+
+// NOTE: Most functions won't use this, as they should use regular api calls. However,
+//  as we are using paths for RequireJS, we explicitly need our local (when serverside).
+async function getPredictController() {
+    let controller: QuerysubControllerBase;
+    if (isNode()) {
+        // NOTE: If we are on node, then the require WON'T run over the network, so we need to use
+        //  our local module path, not the remote one.
+        controller = new QuerysubControllerBase();
+    } else {
+        let nodeId = await querysubNodeId();
+        if (!nodeId) throw new Error("No querysub node found");
+        controller = QuerysubController.nodes[nodeId] as any;
+    }
+    return controller;
+}
+// TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
+//  but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
+//  do too much for us if we already have the fully resolved path...
+//  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
+const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<true> => {
+    let controller = await getPredictController();
+
+    let path = await controller.getModulePath({ functionSpec: spec });
+    setGitURLMapping({ spec, resolvedPath: path });
+    return true;
+});
+
+const getDevFunctionSpecFromCall = cacheJSONArgsEqual(async (call: {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+}) => {
+    let controller = await getPredictController();
+    return controller.getDevFunctionSpecFromCall(call);
+});
+
+
+export function getCallResultPath(call: CallSpec) {
+    return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
+}
+export function getCallResult(call: CallSpec) {
+    return functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId];
+}
+
+/** Force predictions to run in the trigger order, so they can resolve an be added before
+ *      the next predictions. Also, to preserve call order.
+ */
+//const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult | undefined>) => run());
+// UPDATE: We no longer run them in serial, as this caused a lot of lag if we ran a lot of functions which access unique values they needed to sync. Generally speaking, most predictions should run in serial anyway, As most functions should be accessing already synchronized values. 
+const predictRunCommitLoop = (run: () => Promise<PredictResult | undefined>) => run();
+
+
+
+
+// IMPORTANT! This has to be synchronous, and we need to synchronously get into the proxy watcher call. That way other calls after this know to wait for our proxy watcher to finish. Otherwise, it's very, very, very easy to write code, where you call a function, you expect it to write to a value, and then you run some other code, maybe which just uses a commit async, and tries to read from that value. But the value won't exist, because we won't even have started the prediction yet, and the proxy watcher can't possibly know to wait, because we haven't started the prediction. So... this has to be synchronous!
+export function predictCall(call: CallSpec, metadata: FunctionMetadata): {
+    onApplied: () => void;
+    cancel: () => void;
+    predictPromise: Promise<PredictResult | undefined>;
+} {
+    let predictObj = predictCallBase({ call, metadata });
+    let cancel = predictObj.cancel;
+    let onApplied = predictObj.onApplied;
+    registerPredictionBlocker(call.CallId, predictObj.predictPromise);
+    return { cancel, onApplied, predictPromise: predictObj.predictPromise };
+}
+
+function getPredictTime(time: Time) {
+    return { time: time.time, version: predictionLockVersion, creatorId: time.creatorId };
+}
+
+export interface PredictResult {
+    readPaths: Set<string>;
+    readParentPaths: Set<string>;
+    writes: PathValue[];
+    /** Writes that have been replaced AT the write time of the prediction */
+    replacedWriteValues: PathValue[];
+}
+function predictCallBase(config: {
+    call: CallSpec;
+    metadata: FunctionMetadata;
+    overrides?: PathValue[];
+}): {
+    onApplied: () => void;
+    cancel: () => void;
+    predictPromise: Promise<PredictResult | undefined>;
+} {
+    let call = config.call;
+    let pathResultWrite = getCallResultPath(call);
+
+    let actualValueFinished = new PromiseObj();
+    function onActualFinished() {
+        if (actualValueFinished.resolveCalled) return;
+        let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
+        let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
+        if (!result) return;
+        if (result.lastInternalLoopCount === -1) return;
+        cleanupPrediction();
+        actualValueFinished.resolve();
+        for (let callback of onPredictionFinishedCallbacks) {
+            callback({ callId: call.CallId, result, functionId: call.FunctionId });
+        }
+    }
+    function onApplied() {
+        clientWatcher.setWatches({
+            callback: onActualFinished,
+            paths: new Set([pathResultWrite]),
+            parentPaths: new Set(),
+        });
+    }
+
+    let functionResult: FunctionResult = {
+        // Special prediction values (we don't even really need to put anything here, but it
+        //  might as well have the right format, in case a bug causes it to be used somewhere)
+        lastInternalLoopCount: -1,
+        outerLoopCount: -1,
+        totalInternalLoopCount: -1,
+        timeTaken: -1,
+        totalTime: -1,
+        evalTime: 0,
+    };
+
+    // NOTE: We don't cascade predictions, instead only depending on the call succeeding. This is because if we depend on writes,
+    //  we will be invalidated before we get the actual call result, which results in going back in time, a delay, then
+    //  having the actual call results, and so jumping forward again. It is better to just wait, so our latest value
+    //  becomes the latest call value, when we receive it.
+    let predictResultWrite: PathValue = {
+        path: pathResultWrite,
+        value: functionResult,
+        locks: [
+            {
+                // We use a very low negative version, so we don't have to worry about this actually existing. We don't really want
+                //  a lock on startTime, we just want the range lock.
+                startTime: { time: call.runAtTime.time, version: -2344282313, creatorId: call.runAtTime.creatorId },
+                // NOTE: The version is special, and used to detect prediction rejections in some logging.
+                // NOTE: We need to set our end time enough in the future so errors cause us to be rejected
+                endTime: { time: call.runAtTime.time + MAX_CHANGE_AGE, version: Number.MAX_SAFE_INTEGER, creatorId: call.runAtTime.creatorId },
+                path: pathResultWrite,
+                // Technically we are depending on a future time, so... we are depending on ourself? And our value isn't undefined.
+                readIsTransparent: false,
+            }
+        ],
+        lockCount: 1,
+        // We right our result BEFORE the actual result, so it will be clobbered (and before our read lock!)
+        //  when the result arrives (although... it will ALSO be rejected, so... this might be less important...)
+        //  (Also, more importantly, this ha to be UNIQUE, otherwise time locks break!)
+        time: getPredictTime(call.runAtTime),
+        valid: true,
+        event: true,
+    };
+    let predictLocks: ReadLock[] = [
+        {
+            startTime: predictResultWrite.time,
+            endTime: predictResultWrite.time,
+            path: pathResultWrite,
+            readIsTransparent: false,
+        }
+    ];
+    let predictions: PredictResult | undefined;
+    let predictPromise = predictRunCommitLoop(async () => {
+        // IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
+        if (Querysub.DEBUG_PREDICTIONS) {
+            console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
+        }
+
+        let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
+
+        let dryRunResult: {
+            writes: PathValue[];
+            readPaths: Set<string>;
+            readParentPaths: Set<string>;
+        };
+
+        setTimeout(() => {
+            clientWatcher.unwatch(onActualFinished);
+        }, 30 * 1000);
+
+        try {
+            let tempDryRunResult = await Promise.race([
+                await getCallWrites({ call, debugName, overrides: config.overrides, useFinishReordering: true, metadata: config.metadata }),
+                actualValueFinished.promise,
+            ]);
+            if (!tempDryRunResult) {
+                if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
+                    console.log(magenta(`Abort predict call before prediction finished, already received call result`), `${call.DomainName}.${call.FunctionId}`);
+                }
+                return undefined;
+            }
+            dryRunResult = tempDryRunResult;
+        } catch (e: any) {
+            if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
+                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
+            } else {
+                // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
+                //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
+                //  the race to predict the function against the server updating it, it is likely our prediction will now
+                //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
+                //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
+            }
+            return {
+                writes: [],
+                readPaths: new Set(),
+                readParentPaths: new Set(),
+                replacedWriteValues: [],
+            };
+        }
+        predictions = {
+            writes: dryRunResult.writes,
+            readPaths: dryRunResult.readPaths,
+            readParentPaths: dryRunResult.readParentPaths,
+            replacedWriteValues: dryRunResult.writes.map(write => {
+                let path = write.path;
+                let newWrite: PathValue = authorityStorage.getValueAtTime(write.path, write.time) || {
+                    path, valid: true, time: write.time, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true,
+                };
+                newWrite = { ...newWrite };
+                // Use a time very slightly after, so it clobbers the write, and any writes before it, but none after.
+                newWrite.time = write.time;
+                write.time = {
+                    time: write.time.time,
+                    version: write.time.version,
+                    creatorId: write.time.creatorId + 1,
+                };
+                return newWrite;
+            }),
+        };
+
+        // Abort, and don't predict
+        if (didCancel) return predictions;
+
+        for (let value of predictions.writes) {
+            value.source = debugName;
+        }
+
+        // If we already received the actual call, we can't add the prediction, as... the invalidation
+        //      code won't properly immediately reject our prediction, as we are not the authority on the
+        //      path, so it treats it as source of truth.
+        if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
+            if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
+                console.log(magenta(`Abort predict call, already received call result`), `${call.DomainName}.${call.FunctionId}`);
+            }
+            return predictions;
+        }
+
+        // We commit slightly different writes, for  lock invalidation. Our caller doesn't need / doesn't want this
+        //  extra write, so we just don't return it.
+        // NOTE: I guess we technically don't need to actually write to the result, but... might as well?
+        predictions.writes.push(predictResultWrite);
+        // Update all locks to use the same locks as predictResultWrite
+        // Ensure we use the predict time, otherwise these times will be reused when the call actually happens,
+        //  potentially with different values, which breaks all of our ReadLocks!
+        for (let predict of predictions.writes) {
+            predict.time = predictResultWrite.time;
+            predict.locks = predictLocks;
+            predict.lockCount = predictLocks.length;
+        }
+
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: predictions.writes,
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
+        });
+
+        if (Querysub.DEBUG_PREDICTIONS) {
+            console.log(magenta(`Finished and applied predict call`), `${call.DomainName}.${call.FunctionId}`);
+        }
+        return predictions;
+    });
+    logErrors(predictPromise);
+
+    let didCancel = false;
+    function rejectPrediction() {
+        if (didCancel) return;
+        didCancel = true;
+        if (!predictions) return;
+        // Reject our predictions, as the call likely never got committed, so it will never be written
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: predictions.writes.map(write => ({
+                path: write.path,
+                value: write.value,
+                locks: [],
+                lockCount: 0,
+                valid: false,
+                time: predictResultWrite.time,
+                isTransparent: false,
+            })),
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+            doNotArchive: true,
+        });
+    }
+
+    setTimeout(cleanupPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
+    function cleanupPrediction() {
+        if (didCancel) return;
+
+        // ALWAYS reject the prediction, in case the function runner server is down.
+        //  - ALSO, once it goes back up, the write likely will be too far in the future, so our lock won't be rejected,
+        //      will will cause the bad value to stick around.
+        //      We could fix this with an endTime infinitely in the future, but then the lock can never be GCed,
+        //      which create a memory leak.
+        rejectPrediction();
+
+        if (Querysub.AUDIT_PREDICTIONS && predictions) {
+            const afterTime = { time: call.runAtTime.time, version: Number.MAX_SAFE_INTEGER, creatorId: 0 };
+            // Clone predictions, to strip symbols
+            let predictionsCopy = cborEncoder().decode(cborEncoder().encode(predictions)) as typeof predictions;
+            for (let predict of predictionsCopy.writes ?? []) {
+                if (predict.path === pathResultWrite) continue;
+                let finalValueObj = authorityStorage.getValueAtTime(predict.path, afterTime);
+                if (!authorityStorage.isSynced(predict.path)) continue;
+
+                let finalValue = pathValueSerializer.getPathValue(finalValueObj);
+                let predictValue = pathValueSerializer.getPathValue(predict);
+
+                let finalBuffer = cborEncoder().encode(finalValue);
+                let predictBuffer = cborEncoder().encode(predictValue);
+                // If they are different, warn
+                if (!finalBuffer.equals(predictBuffer)) {
+                    // NOTE: This MIGHT be due to delayCommit on another function. We can't know when to force flush
+                    //  delayCommit values, and so we only force flush if other (non-delayCommit) functions are committed
+                    //  in the same schema, which will be wrong if data is accessed cross schema.
+                    //  - If this happens, and is a problem, we COULD handle it, but... it is very difficult, as we would need
+                    //      to evaluate the other function, detect it conflicts with a delayCommit functions, then run the delayCommit
+                    //      function, then re-evaluated the conflicted prediction (which is a lot of communication and reruns
+                    //      between a lot of systems... especially because in a second this will resolve by the server
+                    //      clobbering our prediction).
+                    //      - AND, we can't commit the pending calls until we sort out the order, so this necessarily requires
+                    //          slowing down commits if we are delaying other calls.
+                    console.warn(`${red("Prediction was wrong")}: for ${call.DomainName}.${call.FunctionId} value path ${getPathFromStr(predict.path).join(".")} predict != finalValue. ${config.metadata.delayCommit && "This function is using delay commit. It's likely that you are using the time from Querysub.now() (or accessing the function call time in some other way). This doesn't work with delay commit because the server will rewrite the commit time, instead you should pass the timestamp as a parameter (And then make sure it's not some kind of privilege value that the user could cheat)" || ""} It might be the case that you shouldn't even predict the call client side (you can use the functionMetadata to set it as nopredict). It might also be the case that you do want to predict it, but you should batch the calls, so you only make one call, instead of many calls at once.`, predictValue, finalValue);
+                }
+            }
+        }
+    }
+
+    return {
+        cancel: rejectPrediction,
+        predictPromise,
+        onApplied,
+    };
+}
+
+const getDevFunctionCache = cacheAsyncLimitedJSON(100_000, getDevFunctionSpecFromCall);
+const addToModuleLoaderCache = cacheAsyncLimitedJSON(100_000, addModuleToLoader);
+
+function getFunctionSpec(call: CallSpec): FunctionSpec | undefined {
+    if (!isPublic()) {
+        let obj = getDevFunctionCache({
+            DomainName: call.DomainName,
+            ModuleId: call.ModuleId,
+            FunctionId: call.FunctionId,
+        });
+        if (!obj) throw new Error(`Function not referenced in deploy.ts ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
+        setGitURLMapping({ spec: obj.functionSpec, resolvedPath: obj.modulePath });
+        return obj.functionSpec;
+    }
+
+    let domainObject = functionSchema()[call.DomainName];
+    let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
+    let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
+    if (!functionSpec) {
+        if (!Querysub.isAllSynced()) return undefined;
+        throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
+    }
+    // Add the module to the loader via asking the server the exact url for this call. The loader will
+    //  then load the code from that url when it ends up running it.
+    if (!addToModuleLoaderCache(functionSpec)) return undefined;
+    return functionSpec;
+}
+
+function getFunctionInfoBase(call: CallSpec): {
+    functionSpec: FunctionSpec;
+    baseFunction: Function;
+} | undefined {
+    let functionSpec = getFunctionSpec(call);
+    if (!functionSpec) return undefined;
+
+    let module = getModuleFromConfig(functionSpec);
+    if (module instanceof Promise) {
+        proxyWatcher.triggerOnPromiseFinish(module, { waitReason: `Loading function ${call.DomainName}.${call.ModuleId}.${call.FunctionId}` });
+        return undefined;
+    }
+    let exportPath = getPathFromStr(functionSpec.exportPathStr);
+    let exportObj = module.exports;
+    for (let path of exportPath) {
+        if (!(path in exportObj)) {
+            throw new Error(`Export not found for call prediction: ${path}, in ${call.DomainName}.${call.ModuleId}.${call.FunctionId}. Have ${Object.keys(exportObj)}`);
+        }
+        exportObj = exportObj[path];
+    }
+    let baseFunction = exportObj as Function;
+
+    return { functionSpec, baseFunction };
+}
+
+// IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
+export function getCallWrites(config: {
+    debugName: string;
+    call: CallSpec;
+    overrides?: PathValue[];
+    useFinishReordering?: boolean;
+    metadata?: FunctionMetadata;
+}): Promise<DryRunResult> {
+    let { call, debugName } = config;
+
+    let finishInStartOrder: number | boolean | undefined;
+
+    if (config.useFinishReordering && !config.metadata?.noFinishInStartOrder && call.fromProxy) {
+        let triggerCaller = getCurrentCallCreationProxy();
+        if (!triggerCaller) {
+            require("debugbreak")(2);
+            debugger;
+            throw new Error(`getCallWrites did not happen synchronously when triggering a call. It's very important that we synchronously start the proxy watcher, so we can make sure that predictions happen in a consistent order. FIX THIS! Find the async weighting that was added and move it inside the watch function using cacheAsync to make it watched. For: ${debugName}`);
+        }
+        finishInStartOrder = addEpsilons(triggerCaller.createTime, 1);
+    }
+    let first = true;
+    return proxyWatcher.dryRunFull({
+        debugName,
+        runAtTime: call.runAtTime,
+        overrideAllowLockDomainsPrefixes: [getPathStr1(call.DomainName)],
+        // We are going to clobber locks anyways, so there is no reason to create them
+        unsafeNoLocks: true,
+        overrides: config.overrides,
+        nestedCalls: "inline",
+        // Run after our trigger function. This will insert us before functions which may have been run before us, but after our trigger function. 
+        finishInStartOrder,
+        predictMetadata: config.metadata,
+        watchFunction() {
+            let fncObj = getFunctionInfoBase(call);
+            if (!fncObj) return undefined;
+            let { functionSpec, baseFunction } = fncObj;
+            return overrideCurrentCall({ spec: call, fnc: functionSpec }, () => {
+                if (first) {
+                    if (Querysub.DEBUG_PREDICTIONS) {
+                        console.log(magenta(`Loaded predict function and first run`), `${call.DomainName}.${call.FunctionId}`);
+                    }
+                }
+                first = false;
+
+                let args = parseArgs(call);
+                let result = baseFunction(...args);
+                return result;
+            });
+        },
+    });
 }