querysub 0.340.0 → 0.341.0

package/src/4-querysub/querysubPrediction.ts

@@ -2,13 +2,13 @@ import { cacheJSONArgsEqual, cacheWeak, lazy } from "socket-function/src/caching
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
 import { CLIENTSIDE_PREDICT_LEEWAY, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, compareTime, debugTime, getCreatorId, getNextTime, lockToCallback, predictionLockVersion, timeMinusEpsilon } from "../0-path-value-core/pathValueCore";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
-import { proxyWatcher, atomicObjectRead } from "../2-proxy/PathValueProxyWatcher";
+import { proxyWatcher, atomicObjectRead, DryRunResult, getCurrentCallCreationProxy, debug_getQueueOrder } from "../2-proxy/PathValueProxyWatcher";
 import { getPathFromProxy, getProxyPath } from "../2-proxy/pathValueProxy";
 import { CallSpec, FunctionResult, FunctionSpec, debugCallSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
 import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
 import { logErrors } from "../errors";
-import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2 } from "../path";
-import { Querysub, QuerysubController, QuerysubControllerBase, baseAddCall, callWaitOn, querysubNodeId } from "./QuerysubController";
+import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2, getPathStr3 } from "../path";
+import { Querysub, QuerysubController, QuerysubControllerBase, baseAddCall, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
 import { Benchmark } from "../diagnostics/benchmark";
 import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
 import { runInSerial } from "socket-function/src/batching";
@@ -18,11 +18,14 @@ import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSeriali
 import { setFlag } from "socket-function/require/compileFlags";
 import cbor from "cbor-x";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
-import { isNode, nextId, sort } from "socket-function/src/misc";
+import { PromiseObj, isNode, nextId, sort } from "socket-function/src/misc";
 import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
 import { isLocal } from "../config";
 import { onAllPredictionsFinished } from "../-0-hooks/hooks";
 import { t } from "../2-proxy/schema2";
+import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
+import { addEpsilons } from "../bits";
 setFlag(require, "cbor-x", "allowclient", true);
 const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
 
@@ -46,11 +49,12 @@ async function getPredictController() {
 //  but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
 //  do too much for us if we already have the fully resolved path...
 //  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
-const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
+const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<true> => {
     let controller = await getPredictController();
 
     let path = await controller.getModulePath({ functionSpec: spec });
     setGitURLMapping({ spec, resolvedPath: path });
+    return true;
 });
 
 const getDevFunctionSpecFromCall = cacheJSONArgsEqual(async (call: {
@@ -73,239 +77,42 @@ export function getCallResult(call: CallSpec) {
 /** Force predictions to run in the trigger order, so they can resolve an be added before
  *      the next predictions. Also, to preserve call order.
  */
-const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult>) => run());
+//const predictRunCommitLoop = runInSerial((run: () => Promise<PredictResult | undefined>) => run());
+// UPDATE: We no longer run them in serial, as this caused a lot of lag if we ran a lot of functions which access unique values they needed to sync. Generally speaking, most predictions should run in serial anyway, As most functions should be accessing already synchronized values. 
+const predictRunCommitLoop = (run: () => Promise<PredictResult | undefined>) => run();
 
 
-interface DelayedCall {
-    call: CallSpec;
-    time: number;
-    commit: () => void;
-    cancel: () => void;
-
-    result?: PredictResult;
-}
-
-let callPending: DelayedCall[] = [];
-function flushUpToIncluding(time: number) {
-    while (callPending.length > 0) {
-        if (callPending[0].time > time) break;
-        let call = callPending.shift()!;
-        call.commit();
-    }
-    updateSyncedCount();
-}
 
-export function flushDelayedFunctions() {
-    while (callPending.length > 0) {
-        flushUpToIncluding(callPending[callPending.length - 1].time);
-    }
-}
 
-const pendingState = lazy(() => Querysub.createLocalSchema("querysubPrediction", {
-    count: t.number
-})());
-export function getPendingCountSynced() {
-    return pendingState().count;
-}
-function updateSyncedCount() {
-    Querysub.commitLocal(() => {
-        pendingState().count = callPending.length;
-    });
+// IMPORTANT! This has to be synchronous, and we need to synchronously get into the proxy watcher call. That way other calls after this know to wait for our proxy watcher to finish. Otherwise, it's very, very, very easy to write code, where you call a function, you expect it to write to a value, and then you run some other code, maybe which just uses a commit async, and tries to read from that value. But the value won't exist, because we won't even have started the prediction yet, and the proxy watcher can't possibly know to wait, because we haven't started the prediction. So... this has to be synchronous!
+export function predictCall(call: CallSpec, metadata: FunctionMetadata): {
+    cancel: () => void;
+    predictPromise: Promise<PredictResult | undefined>;
+} {
+    let predictObj = predictCallBase({ call, metadata });
+    let cancel = predictObj.cancel;
+    registerPredictionBlocker(call.CallId, predictObj.predictPromise);
+    return { cancel, predictPromise: predictObj.predictPromise };
 }
 
-
-
-export const addCall = runInSerial(async function addCall(call: CallSpec, metadata: FunctionMetadata) {
-    const nodeId = await querysubNodeId();
-    if (!nodeId) throw new Error("No querysub node found");
-
-    // TODO: Use some heuristics to determine if we should predict the call or not (ex, based on predicted time
-    //  to run, current load, latency to server, time for server to run it, time to sync the necessary data, etc)
-    // TODO: Allow some flags to tell us NOT to predict a call
-    let predict = Querysub.PREDICT_CALLS;
-    if (metadata.nopredict) {
-        predict = false;
-    }
-
-    // NOTE: We predict when call.filterable, as filterable usually just means we are testing
-    //  new functions in development (instead of targeting specific hardware). If we find this
-    //  is annoying, we could add a flag in filterable that explicitly screens out prediction?
-
-    let cancel = () => { };
-
-    if (predict) {
-        let delayed = metadata.delayCommit && !isNode();
-        let predictObj = predictCall({ call, localOnly: delayed });
-        cancel = predictObj.cancel;
-        callWaitOn(call.CallId, predictObj.predictPromise);
-        if (metadata.delayCommit && !isNode()) {
-            let hasRun = false;
-            let delayedCall: DelayedCall = {
-                call,
-                time: Date.now(),
-                commit() {
-                    if (hasRun) return;
-                    hasRun = true;
-                    logErrors(baseAddCall(call, nodeId, cancel, `call (delayed from ${debugTime(call.runAtTime)})`));
-                },
-                cancel() { predictObj.cancel(); },
-            };
-            callPending.push(delayedCall);
-            updateSyncedCount();
-            setTimeout(() => flushUpToIncluding(delayedCall.time), Querysub.DELAY_COMMIT_DELAY);
-            logErrors(predictObj.predictPromise.then(async (prediction): Promise<void> => {
-                prediction = { ...prediction };
-                const resultPrefix = getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results);
-                prediction.writes = prediction.writes.filter(write => !write.path.startsWith(resultPrefix));
-                delayedCall.result = prediction;
-
-                let index = callPending.indexOf(delayedCall);
-                if (index === -1) return;
-                let ourWritePaths = new Set(prediction.writes.map(x => x.path));
-                function candidateCouldBeChangedByWritesSimilarToOurs(candidate: PredictResult) {
-                    return (
-                        candidate.writes.some(write => ourWritePaths.has(write.path))
-                        || candidate.writes.some(write => candidate.readPaths.has(write.path))
-                        || candidate.writes.some(write =>
-                            candidate.readParentPaths.has(getParentPathStr(write.path))
-                            || candidate.readParentPaths.has(write.path)
-                        )
-                    );
-                }
-
-                // NOTE: Technically... we can remove a write if:
-                //  - latest.writes.paths is a superset of candidate.writes.paths
-                //  - (candidate <= intermediate < latest).all(x =>
-                //          test replace writes of x with previous values of x, as time of x
-                //          rerun writes up to and including latest
-                //          verify final resolved writes are equivalent as the final writes as if we had the candidate write
-                //    )
-                // HOWEVER, the last condition is overly arduous to implement, wildly inefficient to calculate, so
-                //      we can simplify it to a more conservative (removes less candidates, but still is never wrong):
-                //  - (candidate < intermediate < latest).all(x => none of x.write.paths are in latest.writes.paths and none of x.read.paths are in latest.writes.paths)
-                //  - candidate.writes.paths is a subset of latest.writes.paths
-                //  -   test replace writes of x with previous values of x, as time of x
-                //      rerun latest
-                //      verify writes2.writes === writes.writes (paths and values)
-                // NOTE: See "delayCommit" in syncSchema for the risks of delaying commits.
-
-                // Find the first write that is a candidate (due to our simplified condition this will only be 1).
-                let curIndex = index - 1;
-                while (curIndex >= 0) {
-                    let candidatePred = callPending[curIndex].result;
-                    if (!candidatePred) {
-                        // NOTE: This shouldn't happen, but if it does... we could just stop searching here, and not
-                        //      remove any predictions.
-                        throw new Error("Predictions finished out of order.");
-                    }
-                    if (candidateCouldBeChangedByWritesSimilarToOurs(candidatePred)) {
-                        break;
-                    }
-                    curIndex--;
-                }
-                // Nothing to remove, we don't overlap any other writes
-                if (curIndex < 0) {
-                    return;
-                }
-
-                let candidate = callPending[curIndex];
-                let candidatePred = candidate.result;
-                if (!candidatePred) {
-                    throw new Error("Predictions finished out of order.");
-                }
-                // candidate writes have to be a strict subset of our writes, otherwise there is no way we will clobber the candidate.
-                if (candidatePred.writes.some(write => !ourWritePaths.has(write.path))) return;
-
-
-                let debugName = `[redundant check]|${call.DomainName}.${call.FunctionId}`;
-                // NOTE: Run right before the call, so we skip it's writes, but use everything else. We could also use
-                //  candidate.time + epsilon, because none of the values between call and candidate matter (they don't write
-                //  to any paths either read from).
-                let beforeCall = { ...call };
-                beforeCall.runAtTime = timeMinusEpsilon(getPredictTime(call.runAtTime));
-                let withoutCandidateResult = await getCallWrites({
-                    call: beforeCall,
-                    debugName,
-                    overrides: candidatePred.replacedWriteValues
-                });
-
-                function areWritesEqual(writes: PathValue[], writes2: PathValue[]) {
-                    if (writes.length !== writes2.length) return false;
-                    // Write order should probably be the same anyways, so... it is far to compare in order
-                    for (let i = 0; i < writes.length; i++) {
-                        let write = writes[i];
-                        let write2 = writes2[i];
-                        if (write.path !== write2.path) return false;
-                        if (pathValueSerializer.compareValuePaths(write, write2) !== 0) {
-                            return false;
-                        }
-                    }
-                    return true;
-                }
-
-                if (!areWritesEqual(withoutCandidateResult.writes, prediction.writes)) {
-                    if (Querysub.DEBUG_CALLS) {
-                        // Failed to collapse delayed functions, because one of the previous predictions was used in the new predicted function. This is valid, but slower than necessary. If pure atomic behavior is not required (which it presumably isn't required, as delayCommit is already being used), try to rewrite the function to be in the form of a set, instead of an add. For example, "function add(count) { x += count; }" => "function set(value) { x = value; }" (calling with set(x + count)).
-                        console.log(`[Querysub] ${yellow("CANNOT DELAY COLLAPSE")}      ${debugCallSpec(candidate.call)} `);
-                    }
-                    return;
-                }
-
-                let index2 = callPending.indexOf(candidate);
-                if (index2 < 0) return;
-                // At this point... we KNOW:
-                //  a) There are no writes between us and `candidate` (writes are never inserted in the middle, only removed,
-                //      so this condition will hold despite our delay)
-                //  b) `candidate` is not required for `call` to run
-                //  c) `call` replaces all values inside of `candidate`
-                // Which means, `candidate is entirely redunant, and can be removed!
-
-                if (Querysub.DEBUG_CALLS) {
-                    console.log(`[Querysub] ${magenta("removed redundant call")}      ${green(debugCallSpec(candidate.call))}    @${debugTime(candidate.call.runAtTime)}`);
-                }
-                callPending.splice(index2, 1);
-                updateSyncedCount();
-                // Cancel as well, in case it isn't redundant, so we can see the issue immediately.
-                candidate.cancel();
-
-                // Move our time back, otherwise repeated delays can result in not committing writes for
-                //  a long time (indefinitely really), which can result in a lot of lost work, and
-                //  has little speed benefit (comitting a write every 5 seconds is fine).
-                delayedCall.time = candidate.time;
-            }));
-
-            if (Querysub.DEBUG_CALLS) {
-                console.log(`[Querysub] ${magenta("delaying")}    ${green(debugCallSpec(call))}    @${debugTime(call.runAtTime)}`);
-            }
-            return;
-        }
-    }
-
-    await flushDelayedFunctions();
-    // NOTE: NO MORE calls can be queued here, because addCall is run in serial, so... we don't need
-    //  to worry about new calls with times > call.runAtTime (and if it happens we'll get a fairly
-    //  clear error clientside, which will show up in a notification, so it is a very safe assumption).
-    await baseAddCall(call, nodeId, cancel, "call");
-});
-
 function getPredictTime(time: Time) {
     return { time: time.time, version: predictionLockVersion, creatorId: time.creatorId };
 }
 
-interface PredictResult {
+export interface PredictResult {
     readPaths: Set<string>;
     readParentPaths: Set<string>;
     writes: PathValue[];
     /** Writes that have been replaced AT the write time of the prediction */
     replacedWriteValues: PathValue[];
 }
-function predictCall(config: {
+function predictCallBase(config: {
     call: CallSpec;
-    localOnly?: boolean;
+    metadata: FunctionMetadata;
     overrides?: PathValue[];
 }): {
     cancel: () => void;
-    predictPromise: Promise<PredictResult>;
+    predictPromise: Promise<PredictResult | undefined>;
 } {
     let call = config.call;
     let pathResultWrite = getCallResultPath(call);
@@ -357,15 +164,14 @@ function predictCall(config: {
     ];
     let predictions: PredictResult | undefined;
     let predictPromise = predictRunCommitLoop(async () => {
+        // IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
         if (Querysub.DEBUG_PREDICTIONS) {
             console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
         }
 
-        if (!config.localOnly) {
-            // Watch the result, so we know when our prediction is rejected (which will be as soon as the result
-            //  has a real value).
-            remoteWatcher.watchLatest({ paths: [pathResultWrite], parentPaths: [] });
-        }
+        // Watch the result, so we know when our prediction is rejected (which will be as soon as the result
+        //  has a real value).
+        remoteWatcher.watchLatest({ paths: [pathResultWrite], parentPaths: [] });
         let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
 
         let dryRunResult: {
@@ -373,25 +179,52 @@ function predictCall(config: {
             readPaths: Set<string>;
             readParentPaths: Set<string>;
         };
+
+        let actualValueFinished = new PromiseObj();
+        function onActualFinished() {
+            if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
+                actualValueFinished.resolve();
+            }
+        }
+        clientWatcher.setWatches({
+            callback: onActualFinished,
+            paths: new Set([pathResultWrite]),
+            parentPaths: new Set(),
+        });
         try {
-            dryRunResult = await getCallWrites({ call, debugName, overrides: config.overrides });
-        } catch (e: any) {
-
-            if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
-                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
-            } else {
-                // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
-                //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
-                //  the race to predict the function against the server updating it, it is likely our prediction will now
-                //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
-                //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
+
+            try {
+                let tempDryRunResult = await Promise.race([
+                    await getCallWrites({ call, debugName, overrides: config.overrides, useFinishReordering: true, metadata: config.metadata }),
+                    actualValueFinished.promise,
+                ]);
+                if (!tempDryRunResult) {
+                    if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
+                        console.log(magenta(`Abort predict call before prediction finished, already received call result`), `${call.DomainName}.${call.FunctionId}`);
+                    }
+                    return undefined;
+                }
+                dryRunResult = tempDryRunResult;
+            } catch (e: any) {
+
+                if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
+                    console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
+                } else {
+                    // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
+                    //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
+                    //  the race to predict the function against the server updating it, it is likely our prediction will now
+                    //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
+                    //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
+                }
+                return {
+                    writes: [],
+                    readPaths: new Set(),
+                    readParentPaths: new Set(),
+                    replacedWriteValues: [],
+                };
             }
-            return {
-                writes: [],
-                readPaths: new Set(),
-                readParentPaths: new Set(),
-                replacedWriteValues: [],
-            };
+        } finally {
+            clientWatcher.unwatch(onActualFinished);
         }
         predictions = {
             writes: dryRunResult.writes,
@@ -422,7 +255,7 @@ function predictCall(config: {
         //      code won't properly immediately reject our prediction, as we are not the authority on the
         //      path, so it treats it as source of truth.
         if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
-            if (Querysub.DEBUG_PREDICTIONS) {
+            if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
                 console.log(magenta(`Abort predict call, already received call result`), `${call.DomainName}.${call.FunctionId}`);
             }
             return predictions;
@@ -451,7 +284,7 @@ function predictCall(config: {
         lockToCallback.watchLock(predictLocks[0], predictions.writes);
 
         if (Querysub.DEBUG_PREDICTIONS) {
-            console.log(magenta(`Finished predict call`), `${call.DomainName}.${call.FunctionId}`);
+            console.log(magenta(`Finished and applied predict call`), `${call.DomainName}.${call.FunctionId}`);
         }
         return predictions;
     });
@@ -508,7 +341,7 @@ function predictCall(config: {
                     //      clobbering our prediction).
                     //      - AND, we can't commit the pending calls until we sort out the order, so this necessarily requires
                     //          slowing down commits if we are delaying other calls.
-                    console.warn(`${red("Prediction was wrong")}: ${getPathFromStr(predict.path).join(".")} predict != finalValue`, predictValue, finalValue);
+                    console.warn(`${red("Prediction was wrong")}: for ${call.DomainName}.${call.FunctionId} value path ${getPathFromStr(predict.path).join(".")} predict != finalValue. ${config.metadata.delayCommit && "This function is using delay commit. It's likely that you are using the time from Querysub.now() (or accessing the function call time in some other way). This doesn't work with delay commit because the server will rewrite the commit time, instead you should pass the timestamp as a parameter (And then make sure it's not some kind of privilege value that the user could cheat)" || ""} It might be the case that you shouldn't even predict the call client side (you can use the functionMetadata to set it as nopredict). It might also be the case that you do want to predict it, but you should batch the calls, so you only make one call, instead of many calls at once.`, predictValue, finalValue);
                 }
             }
         }
@@ -520,43 +353,46 @@ function predictCall(config: {
     };
 }
 
-export async function getCallWrites(config: {
-    debugName: string;
-    call: CallSpec;
-    overrides?: PathValue[];
-}) {
-    let { call, debugName } = config;
+const getDevFunctionCache = cacheAsyncLimitedJSON(100_000, getDevFunctionSpecFromCall);
+const addToModuleLoaderCache = cacheAsyncLimitedJSON(100_000, addModuleToLoader);
 
-    let functionSpec: FunctionSpec | undefined;
+function getFunctionSpec(call: CallSpec): FunctionSpec | undefined {
     if (isLocal()) {
-        let obj = await getDevFunctionSpecFromCall({
+        let obj = getDevFunctionCache({
             DomainName: call.DomainName,
             ModuleId: call.ModuleId,
             FunctionId: call.FunctionId,
         });
         if (!obj) throw new Error(`Function not referenced in deploy.ts ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
-        functionSpec = obj.functionSpec;
-        setGitURLMapping({ spec: functionSpec, resolvedPath: obj.modulePath });
-    } else {
-        const obj = await proxyWatcher.commitFunction({
-            watchFunction: function getModuleConfig() {
-                let domainObject = functionSchema()[call.DomainName];
-                let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
-                let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
-                return { functionSpec };
-            }
-        });
-        functionSpec = obj.functionSpec;
+        setGitURLMapping({ spec: obj.functionSpec, resolvedPath: obj.modulePath });
+        return obj.functionSpec;
+    }
 
-        if (!functionSpec) {
-            throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
-        }
-        // Add the module to the loader via asking the server the exact url for this call. The loader will
-        //  then load the code from that url when it ends up running it.
-        await addModuleToLoader(functionSpec);
+    let domainObject = functionSchema()[call.DomainName];
+    let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
+    let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
+    if (!functionSpec) {
+        if (!Querysub.isAllSynced()) return undefined;
+        throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
     }
+    // Add the module to the loader via asking the server the exact url for this call. The loader will
+    //  then load the code from that url when it ends up running it.
+    if (!addToModuleLoaderCache(functionSpec)) return undefined;
+    return functionSpec;
+}
 
-    let module = await getModuleFromConfig(functionSpec);
+function getFunctionInfoBase(call: CallSpec): {
+    functionSpec: FunctionSpec;
+    baseFunction: Function;
+} | undefined {
+    let functionSpec = getFunctionSpec(call);
+    if (!functionSpec) return undefined;
+
+    let module = getModuleFromConfig(functionSpec);
+    if (module instanceof Promise) {
+        proxyWatcher.triggerOnPromiseFinish(module, { waitReason: `Loading function ${call.DomainName}.${call.ModuleId}.${call.FunctionId}` });
+        return undefined;
+    }
     let exportPath = getPathFromStr(functionSpec.exportPathStr);
     let exportObj = module.exports;
     for (let path of exportPath) {
@@ -567,8 +403,32 @@ export async function getCallWrites(config: {
     }
     let baseFunction = exportObj as Function;
 
-    let specTyped = functionSpec;
-    return await proxyWatcher.dryRunFull({
+    return { functionSpec, baseFunction };
+}
+
+// IMPORTANT! See the addCall note for why we have to NOT have any waiting before getCallWrites.
+export function getCallWrites(config: {
+    debugName: string;
+    call: CallSpec;
+    overrides?: PathValue[];
+    useFinishReordering?: boolean;
+    metadata?: FunctionMetadata;
+}): Promise<DryRunResult> {
+    let { call, debugName } = config;
+
+    let finishInStartOrder: number | boolean | undefined;
+
+    if (config.useFinishReordering) {
+        let triggerCaller = getCurrentCallCreationProxy();
+        if (!triggerCaller) {
+            require("debugbreak")(2);
+            debugger;
+            throw new Error(`getCallWrites did not happen synchronously when triggering a call. It's very important that we synchronously start the proxy watcher, so we can make sure that predictions happen in a consistent order. FIX THIS! Find the async weighting that was added and move it inside the watch function using cacheAsync to make it watched. For: ${debugName}`);
+        }
+        finishInStartOrder = addEpsilons(triggerCaller.createTime, 1);
+    }
+    let first = true;
+    return proxyWatcher.dryRunFull({
         debugName,
         runAtTime: call.runAtTime,
         overrideAllowLockDomainsPrefixes: [getPathStr1(call.DomainName)],
@@ -576,10 +436,24 @@ export async function getCallWrites(config: {
         unsafeNoLocks: true,
         overrides: config.overrides,
         nestedCalls: "inline",
+        // Run after our trigger function. This will insert us before functions which may have been run before us, but after our trigger function. 
+        finishInStartOrder,
+        predictMetadata: config.metadata,
         watchFunction() {
-            return overrideCurrentCall({ spec: call, fnc: specTyped }, () => {
+            let fncObj = getFunctionInfoBase(call);
+            if (!fncObj) return undefined;
+            let { functionSpec, baseFunction } = fncObj;
+            return overrideCurrentCall({ spec: call, fnc: functionSpec }, () => {
+                if (first) {
+                    if (Querysub.DEBUG_PREDICTIONS) {
+                        console.log(magenta(`Loaded predict function and first run`), `${call.DomainName}.${call.FunctionId}`);
+                    }
+                }
+                first = false;
+
                 let args = parseArgs(call);
-                return baseFunction(...args);
+                let result = baseFunction(...args);
+                return result;
             });
         },
     });

package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx

@@ -81,7 +81,7 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     await sendEmail({
         to: notifyEmails,
         fromPrefix: "error-digest",
-        subject: `${errorCount} errors | ${formatNumber(failingFiles)} failing files | ${warningCount} warnings${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalCompressedBytes)} / ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} files`,
+        subject: `${errorCount} err | >= ${formatNumber(failingFiles)} lines | ${warningCount} warn${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${suppressedErrors + suppressedWarnings} suppressed | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalCompressedBytes)} / ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} log files`,
         contents: <div>
             <h2>Error Summary</h2>
             <ul style="list-style-type: none; padding-left: 0;">

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -128,6 +128,7 @@ const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[
         if (countFiltered > 0) {
             message += `\n+${countFiltered} more errors`;
         }
+        console.log(`Discord message: ${message}`);
         void sendDiscordMessage({
             webhookURL,
             message,

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -110,6 +110,11 @@ DEBUG: Why we would get into an infinite identify loop:
 IMPORTANT! The infinite identify loop causes servers to never finish identification, and so new PathValueServers couldn't start. SO THIS IS A REALLY BIG ISSUE!
     - Restarting the server fixed it. I'm not sure if it was one service, or all of them?
 
+DEBUG: Backblaze errors
+    - Add a life cycle which handles the fact that one request might result in errors and retries. It also might result in an initial connection being done. It might be waiting on the initial connection, or it might be the one that triggers the initial connection.
+    - Being able to split up the different life cycles by various values, such as if they had to connect, if they had to retry, etc., would be useful as well. And then also splitting up the timing. And graphing various timing such as how long the request actually takes (minus the retries, initial connection, throttling, etc), how long the overall call takes, etc.
+    - And being able to plot by time would be useful as well, so we can see if the errors are time-based. 
+
 SPECIAL UI links for certain errors in log view
     - Probably dynamically created, based on contents of log
     - LINKS to filters for all these special errors on a special page

package/src/functional/promiseCache.ts

@@ -32,13 +32,22 @@ export function cacheAsyncLimited<Arg, Return>(limit: number, fnc: (arg: Arg) =>
 }
 
 export const cacheAsyncSynced = cacheAsyncLimitedJSON;
-export function cacheAsyncLimitedJSON<Arg, Return>(limit: number, fnc: (arg: Arg) => Promise<Return>) {
+export function cacheAsyncLimitedJSON<Arg, Return>(
+    limit: number,
+    fnc: (arg: Arg) => Promise<Return>
+): ((arg: Arg) => Return | undefined) & {
+    clear(): void;
+    promise: (arg: Arg) => Promise<Return>;
+} {
     let results = new Map<string, { type: "result"; value: Return } | { type: "error"; error: Error; }>();
     let promiseValues = cacheLimited(limit, (json: string) => fnc(JSON.parse(json)));
     get["clear"] = () => {
         results.clear();
         promiseValues.clear();
     };
+    get.promise = (arg: Arg) => {
+        return promiseValues(JSON.stringify(arg));
+    };
     return get;
     function get(arg: Arg) {
         let json = JSON.stringify(arg);