querysub 0.340.0 → 0.341.0

package/src/4-dom/qreact.tsx

@@ -531,6 +531,7 @@ class QRenderClass {
             void Promise.resolve().finally(() => {
                 logErrors(proxyWatcher.commitFunction({
                     canWrite: true,
+                    baseFunction: callback,
                     watchFunction() {
                         QRenderClass.renderingComponentId = self.id;
                         try {
@@ -555,7 +556,13 @@ class QRenderClass {
 
         let statics = vNode.type as QComponentStatic;
         // NOTE: This used to exist in an `atomicWrites: true` runOnce, but... I don't think that is required anymore
-        let instance = new vNode.type(config.props) as preact.Component<{ [key: string]: unknown }>;
+        let instance: preact.Component<{ [key: string]: unknown }>;
+        QRenderClass.renderingComponentId = this.id;
+        try {
+            instance = new vNode.type(config.props) as preact.Component<{ [key: string]: unknown }>;
+        } finally {
+            QRenderClass.renderingComponentId = undefined;
+        }
         (instance as any)[idSymbol] = this.id;
         if (isValueProxy(instance)) {
             // This will probably result in an error later on. It is usually due to us failing to identify fragments.
@@ -616,6 +623,8 @@ class QRenderClass {
 
         self.instance.componentWillMount && logErrors(proxyWatcher.commitFunction({
             debugName: getDebugName("componentWillMount"),
+            // eslint-disable-next-line @typescript-eslint/unbound-method
+            baseFunction: self.instance.componentWillMount,
             watchFunction() {
                 contextCommit(() => {
                     self.instance.componentWillMount?.();
@@ -1175,6 +1184,7 @@ class QRenderClass {
                         void Promise.resolve().finally(() => {
                             logErrors(proxyWatcher.commitFunction({
                                 canWrite: true,
+                                baseFunction: ref,
                                 watchFunction() {
                                     ref(node);
                                 },
@@ -1219,6 +1229,7 @@ class QRenderClass {
                                 void Promise.resolve().finally(() => {
                                     logErrors(proxyWatcher.commitFunction({
                                         canWrite: true,
+                                        baseFunction: ref2,
                                         watchFunction() {
                                             ref2(node);
                                         },

package/src/4-querysub/Querysub.ts

@@ -190,6 +190,7 @@ export class Querysub {
      *  - NOTE: If this is too short, predictions stop before the server can finish. If it's too long,
      *      server issues (such as the FunctionRunner being missing) won't be noticed. So...
      *      we SHOULD increase this when our setup is more stable (60 seconds is probably good).
+     *  SHOULD be > DELAY_COMMIT_DELAY, Otherwise delayed commits are going to cause predictions to be rejected before we even submit the actual call. 
     */
     public static PREDICTION_MAX_LIFESPAN = 1000 * 15;
 
@@ -292,7 +293,7 @@ export class Querysub {
         explicitlyTrigger: () => void;
     } & SyncWatcher {
         return proxyWatcher.createWatcher({
-            debugName: watcher.name,
+            baseFunction: watcher,
             canWrite: true,
             ...options,
             watchFunction: () => {
@@ -336,15 +337,18 @@ export class Querysub {
             process.exit();
         }
     };
-
     public static syncedCommit = Querysub.serviceWrite;
     public static commitSynced = Querysub.serviceWrite;
     public static commitAsync = Querysub.serviceWrite;
+    /** IMPORTANT! Functions called with commitAsync will finish in their call order, allowing you to use this to read remote values and commit to either remote state or local state, and have the writes be applied in the same order as if they all happened in serial. If you don't want this, or don't need this, you can either use Querysub.commit (If you don't need to wait for it to finish or access the result), or you can pass { finishInStartOrder: false }
+     *  - ONLY DEFAULTED WHEN isClient()
+     */
     public static async serviceWrite<T>(fnc: () => T, options?: Partial<WatcherOptions<T>>) {
         let calls: CallSpec[] = [];
         let result = await proxyWatcher.commitFunction({
             canWrite: true,
             watchFunction: fnc,
+            finishInStartOrder: isClient(),
             onCallCommit(call, metadata) {
                 calls.push(call);
             },
@@ -403,6 +407,8 @@ export class Querysub {
     public static isAllSynced = Querysub.allSynced;
     public static isAnyUnsynced = Querysub.anyUnsynced;
 
+    public static isInWatcher = () => !!proxyWatcher.getTriggeredWatcherMaybeUndefined();
+
     public static onNextDisconnect = SocketFunction.onNextDisconnect;
     public static isNodeConnected = SocketFunction.isNodeConnected;
 
@@ -499,6 +505,20 @@ export class Querysub {
         });
     }
 
+    /** Returns true if any predictions are running. In which case, we will correctly rerun the function and consider it non-synced until the predictions finish. This allows you to check for predictions of the in your function and return if you have any. This is useful for functions that need a stable state before they run. However, if many functions use this and you trigger them at once, it might result in an n-squared situation, so this should be used with caution.
+     *  - The best use case is if something is triggering maybe on focus and on blur, and you want to make sure the on blur changes happen before your on focus changes happen.
+     */
+    public static syncAnyPredictionsPending(): boolean {
+        let waitPromise = onAllPredictionsFinished();
+        if (waitPromise) {
+            proxyWatcher.triggerOnPromiseFinish(waitPromise, {
+                waitReason: "Waiting for predictions to finish",
+            });
+            return true;
+        }
+        return false;
+    }
+
     public static onCallPredict = (x: CallSpec | undefined) => onCallPredict(x);
     /** NOTE: USUALLY you don't have to wait for predictions, as functions will run in order anyways.
      *      BUT, if you run a synced function in the UI, then somewhere unrelated try to read values
@@ -1127,11 +1147,15 @@ let initInterval = cache((interval: number) => {
 });
 
 function getSetTime() {
+    let watcher = proxyWatcher.getTriggeredWatcherMaybeUndefined();
+    if (watcher?.options.predictMetadata?.delayCommit) {
+        debugger;
+        console.warn(`Accessed the function time inside of a delayed commit call in ${watcher.debugName}. This means the prediction is almost certainly going to be wrong. You should provide Date.now() as an argument to the function instead. `);
+    }
     let call = getCurrentCallAllowUndefined();
     if (call) {
         return call.runAtTime.time;
     }
-    let watcher = proxyWatcher.getTriggeredWatcherMaybeUndefined();
     if (watcher?.options.temporary) {
         return watcher.createTime;
     }
@@ -1313,4 +1337,5 @@ import { MachineController } from "../deployManager/machineController";
 import { getRecords, setRecord } from "../-b-authorities/dnsAuthority";
 import { testTCPIsListening } from "socket-function/src/networking";
 import { getNodeId, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { onAllPredictionsFinished } from "../-0-hooks/hooks";
 

package/src/4-querysub/QuerysubController.ts

@@ -44,6 +44,7 @@ import { mergeFilterables, parseFilterable, serializeFilterable } from "../misc/
 import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
 import { getDomain, isBootstrapOnly, isLocal } from "../config";
 import { logDisk } from "../diagnostics/logs/diskLogger";
+import { flushPredictionQueueBase, runInPredictionQueue, syncHasPendingPredictionsBase } from "./predictionQueue";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
@@ -148,7 +149,7 @@ let pendingPredictedCalls = new Map<string, {
 }>();
 export const debug_pendingPredictedCalls = pendingPredictedCalls;
 
-export function callWaitOn(callId: string, promise: Promise<unknown>) {
+export function registerPredictionBlocker(callId: string, promise: Promise<unknown>) {
     let waitObj = pendingPredictedCalls.get(callId);
     if (!waitObj) return;
     let seqNum = ++waitObj.seqNum;
@@ -164,34 +165,72 @@ writeCall.value = async (callSpec: CallSpec, metadata: FunctionMetadata) => {
         obj: new PromiseObj(),
         seqNum: 0,
     });
-    let promise = (async () => {
-        // IMPORTANT! While it may seem like we can use addCall even if we are trusted, so we can
-        //  have function calling prediction, we can't. If did then we would depend on these
-        //  predictions with our writes, which could end up on other nodes, which would then
-        //  be rejected (as our predictions use a Time.version that is only on our
-        //  node). The in unavoidable, as it is too difficult to have predictions, but also not have
-        //  predictions, on the same node.
-        //  - AND, we can't just use the same Time.version as the FunctionRunner will use, as our
-        //      ValuePaths might differ, which would sometimes work, BUT, other times our ReadLocks
-        //      would be valid, BUT, the actual read value would be different (as we might compute
-        //      a different value than FunctionRunner). So... we would lose locking, at which point,
-        //      why even use FunctionRunner in the first place! Just write the writes directly!
-        // TODO: Add the ability to use direct writes BUT if we have a rejection, commit it as a regular function call
-        //  - This will be a lot more efficient, although, if the committed dies and the function is rejected, the
-        //      value will be lost, which isn't great. However... trusted nodes shouldn't be dying so easily!
-        let isTrusted = isServer() && await isTrustedByDomain(callSpec.DomainName);
-        if (isTrusted) {
-            // NOTE: Direct writes should probably be used for calls from trusted nodes, which will
-            //  be a lot faster.
-            await baseWriteCall(callSpec, metadata);
-        } else {
-            await addCall(callSpec, metadata);
+    // IMPORTANT! While it may seem like we can use addCall even if we are trusted, so we can
+    //  have function calling prediction, we can't. If did then we would depend on these
+    //  predictions with our writes, which could end up on other nodes, which would then
+    //  be rejected (as our predictions use a Time.version that is only on our
+    //  node). The in unavoidable, as it is too difficult to have predictions, but also not have
+    //  predictions, on the same node.
+    //  - AND, we can't just use the same Time.version as the FunctionRunner will use, as our
+    //      ValuePaths might differ, which would sometimes work, BUT, other times our ReadLocks
+    //      would be valid, BUT, the actual read value would be different (as we might compute
+    //      a different value than FunctionRunner). So... we would lose locking, at which point,
+    //      why even use FunctionRunner in the first place! Just write the writes directly!
+    // TODO: Add the ability to use direct writes BUT if we have a rejection, commit it as a regular function call
+    //  - This will be a lot more efficient, although, if the committed dies and the function is rejected, the
+    //      value will be lost, which isn't great. However... trusted nodes shouldn't be dying so easily!
+    let isTrusted = isServer() && await isTrustedByDomain(callSpec.DomainName);
+    if (isTrusted) {
+        // NOTE: Direct writes should probably be used for calls from trusted nodes, which will
+        //  be a lot faster.
+        let promise = baseWriteCall(callSpec, metadata);
+        registerPredictionBlocker(callSpec.CallId, promise);
+        await promise;
+    } else if (!Querysub.PREDICT_CALLS || metadata.nopredict) {
+        let promise = baseAddCall(callSpec);
+        registerPredictionBlocker(callSpec.CallId, promise);
+        await promise;
+    } else {
+        let success = false;
+        let cancel = () => { };
+        try {
+            let obj = prediction.predictCall(callSpec, metadata);
+            cancel = obj.cancel;
+
+            // NOTE: All functions go through the prediction queue, because all functions have to be in order whether or not they're delayed. If we add a non-delayed function to the queue, it's going to cause the other functions to finish as soon as possible, which is going to be when their predictions are ready. 
+            let result = await runInPredictionQueue({
+                callSpec,
+                metadata,
+                prediction: obj.predictPromise,
+            });
+
+            if (result === "run") {
+                // NOTE: I think awaiting here is fine, as the the prediction blockers don't wait based on the right call finishing, and I don't think anything really waits for this to finish. 
+                await baseAddCall(callSpec);
+            } else {
+                cancel();
+            }
+            success = true;
+        } finally {
+            if (!success) {
+                // Cancelling is important, as if it never got committed to the server, we won't get a rejection for the prediction. So we just have to abort the prediction right away, or otherwise it will stay forever and cause bugs. 
+                cancel();
+            }
         }
-    })();
-    callWaitOn(callSpec.CallId, promise);
-    await promise;
+    }
 };
 
+
+export async function flushDelayedFunctions() {
+    await flushPredictionQueueBase();
+}
+export function syncHasPendingPredictions() {
+    return syncHasPendingPredictionsBase();
+}
+
+
+
+
 export async function onCallPredict(call: CallSpec | undefined) {
     if (!call) return;
     await pendingPredictedCalls.get(call.CallId)?.obj.promise;
@@ -215,17 +254,10 @@ onAllPredictionsFinished.declare(() => {
 });
 
 
-export async function flushDelayedFunctions() {
-    await prediction.flushDelayedFunctions();
-}
-
-export async function addCall(call: CallSpec, metadata: FunctionMetadata) {
-    await prediction.addCall(call, metadata);
-}
 
-export async function baseAddCall(call: CallSpec, nodeId: string, cancel: () => void, type: string) {
+export async function baseAddCall(call: CallSpec) {
     if (Querysub.DEBUG_CALLS) {
-        console.log(`[Querysub] ${magenta(type)}    ${green(debugCallSpec(call))}    @${debugTime(call.runAtTime)}`);
+        console.log(`[Querysub] ${magenta("call")}    ${green(debugCallSpec(call))}    @${debugTime(call.runAtTime)}`);
     }
     let errorWatcher = Querysub.createWatcher(() => {
         let callResult = atomic(prediction.getCallResult(call));
@@ -239,14 +271,10 @@ export async function baseAddCall(call: CallSpec, nodeId: string, cancel: () =>
     setTimeout(() => {
         errorWatcher.dispose();
     }, MAX_CHANGE_AGE);
-    try {
-        await QuerysubController.nodes[nodeId].addCall(call);
-    } catch (e) {
-        // Cancellation is IMPORTANT! Otherwise the prediction might never be rejected,
-        //  and so the client will keep stale data!
-        cancel();
-        throw e;
-    }
+
+    const nodeId = await querysubNodeId();
+    if (!nodeId) throw new Error("No querysub node found");
+    await QuerysubController.nodes[nodeId].addCall(call);
 }
 
 export class QuerysubControllerBase {

package/src/4-querysub/predictionQueue.tsx

@@ -0,0 +1,183 @@
+import { timeInSecond } from "socket-function/src/misc";
+import { CallSpec, debugCallSpec, functionSchema } from "../3-path-functions/PathFunctionRunner";
+import { FunctionMetadata } from "../3-path-functions/syncSchema";
+import { getParentPathStr, getPathFromStr, getPathStr2 } from "../path";
+import { PromiseObj } from "../promise";
+import { Querysub } from "./QuerysubController";
+import { PredictResult } from "./querysubPrediction";
+import { delay } from "socket-function/src/batching";
+import { blue, green, magenta, yellow } from "socket-function/src/formatting/logColors";
+import { formatTime } from "socket-function/src/formatting/format";
+import { measureBlock, measureWrap } from "socket-function/src/profiling/measure";
+import { debugTime } from "../0-path-value-core/pathValueCore";
+import { t } from "./Querysub";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+
+let commitQueues: Map<string, {
+    callSpec: CallSpec;
+    metadata: FunctionMetadata;
+    prediction: Promise<PredictResult | undefined>;
+    predictionResult: PredictResult | undefined;
+    commitTime: PromiseObj<void>;
+    resolveOrderedPromise: PromiseObj<void>;
+    canBeClobbered: boolean;
+}[]> = new Map();
+let commitQueueCount = Querysub.createLocalSchema("commitQueueCount", {
+    count: t.number,
+});
+
+export async function runInPredictionQueue(config: {
+    callSpec: CallSpec;
+    metadata: FunctionMetadata;
+    prediction: Promise<PredictResult | undefined>;
+}): Promise<"cancel" | "run"> {
+    let { callSpec, prediction, metadata } = config;
+
+    let namespace = getPathStr2(callSpec.DomainName, callSpec.ModuleId);
+    // NOTE: While in an ideal world every single function ID would be unique, realistically functions in the same module are likely to access the same data. So by breaking up by module ID, we get a lot of usefulness by making it so that delays often can queue up despite unrelated functions running. But there aren't also many issues due to the delays because things still operate in a reasonable order. 
+    let commitQueue = commitQueues.get(namespace);
+    if (!commitQueue) {
+        commitQueue = [];
+        commitQueues.set(namespace, commitQueue);
+    }
+
+    let commitTime = new PromiseObj<void>();
+    if (metadata.delayCommit) {
+        setTimeout(() => {
+            commitTime.resolve();
+        }, Querysub.DELAY_COMMIT_DELAY);
+    } else if (commitQueue.length > 0) {
+        // If we have anything that's delaying, always wait a few milliseconds. This might become very annoying later. However, I assume due to network latency, five milliseconds is not going to matter. And this will help batch values more. So even if we do have many non-delayed values that are running, hopefully we should get the predictions done pretty fast, and we should still be able to remove some clobbered values. 
+        setTimeout(() => {
+            commitTime.resolve();
+        }, 5);
+    } else {
+        commitTime.resolve();
+    }
+
+
+    let obj = {
+        callSpec,
+        metadata,
+        prediction,
+        predictionResult: undefined as PredictResult | undefined,
+        commitTime,
+        resolveOrderedPromise: new PromiseObj<void>(),
+        canBeClobbered: !!metadata.delayCommit,
+    };
+    void prediction.then(result => {
+        obj.predictionResult = result;
+    });
+    commitQueue.push(obj);
+    Querysub.localCommit(() => commitQueueCount().count++);
+
+    let stillWaiting = true;
+    void (async () => {
+        let startTime = Date.now();
+        while (stillWaiting) {
+            await delay(timeInSecond * 5);
+            if (!stillWaiting) break;
+            let duration = Date.now() - startTime;
+            console.warn(`VERY SLOW PREDICTION FUNCTION! While this is pending, we cannot commit other writes in this namespace     ${debugCallSpec(callSpec)}    has been running for ${formatTime(duration)}`);
+        }
+    })();
+    let ourPredictBase: PredictResult | undefined;
+    try {
+        ourPredictBase = await prediction;
+    } finally {
+        stillWaiting = false;
+    }
+    const ourPredict = ourPredictBase;
+
+    await commitTime.promise;
+
+    // NOTE: I believe, due to the commit time, this should result in batches where we have many values clobbered by one value, and then after that many values clobbered by a new value, etc. etc. But I guess we'll find out...
+
+    let status: "cancel" | "run" = "run";
+    let index = commitQueue.indexOf(obj);
+    if (index >= 0 && obj.canBeClobbered && ourPredict) {
+        // NOTE: Usually, the predictions should be much faster than the times, so most of the predictions after us should be ready.
+
+        let isIncrementTypeValue = (
+            ourPredict.writes.find(x => ourPredict.readPaths.has(x.path))
+            || ourPredict.writes.find(x => ourPredict.readParentPaths.has(getParentPathStr(x.path)))
+        );
+        // if (isIncrementTypeValue) {
+        //     console.warn(`A delayCommit call Is writing a value that is also reading, which means it can never clobber itself. Suggest it that you either remove the delayCommit flag as it's not doing anything, or change it so that you set the value instead of incrementing it. Fnc: ${debugCallSpec(callSpec)}`, { isIncrementTypeValue });
+        // }
+        // NOTE: Actually, we always end up reading and writing to the same values, because we check values before we write to them, so we don't write identical values. And even if we don't, there's too much code which is focused on initialization, so... We just won't do this check, and if it causes problems, we'll probably notice pretty quickly in the UI. 
+        isIncrementTypeValue = undefined;
+
+        if (!isIncrementTypeValue) {
+            let examplePath0 = getProxyPath(() => functionSchema()[""].PathFunctionRunner);
+            let examplePath2 = getProxyPath(() => functionSchema()[""].PathFunctionRunner[""].Results);
+            let path0Array = getPathFromStr(examplePath0);
+            let path2Array = getPathFromStr(examplePath2);
+            let value0 = path0Array[path0Array.length - 1];
+            let index0 = path0Array.length - 1;
+            let value2 = path2Array[path2Array.length - 1];
+            let index2 = path2Array.length - 1;
+
+            // We only need to see if word clobbered values should unwind in relative order. 
+            const doesValueClobberUs = measureWrap(function doesValueClobberUs(value: PredictResult): boolean {
+                // NOTE: See the delay commit notes for the cases that this doesn't handle. However, it should handle most cases, so it's probably fine. And if it doesn't handle a case, then we will see fairly quickly because the when the predictions are cancelled, we'll see values change that shouldn't (in the UI... hopefully).
+                let otherWrites = new Set(value.writes.map(write => write.path));
+                let nonFunctionWrites = ourPredict.writes.filter(write => {
+                    let arr = getPathFromStr(write.path);
+                    let isFunctionWrite = (
+                        arr[index0] === value0 && arr[index2] === value2
+                    );
+                    return !isFunctionWrite;
+                });
+                let differentWrite = nonFunctionWrites.find(write => !otherWrites.has(write.path));
+                return (
+                    // If it fully writes all the values we write to, then it eclipses us. And if it eclipses us And we aren't an increment type commit, then it likely clobbers us (As if we aren't an increment type commit, Then it is possible we are fully isolated with none of our reads changing, In which case, if we are eclipsed then by a value that also is like that, then clobbering is safe).
+                    !differentWrite
+                );
+            });
+
+            // NOTE: This is quadratic, but I think it should be fine. If not, we have the measure wrap, so we'll know. 
+            for (let i = commitQueue.length - 1; i > index; i--) {
+                let value = commitQueue[i].predictionResult;
+                if (!value) continue;
+                if (doesValueClobberUs(value)) {
+                    // Don't allow it to be clobbered. Otherwise, we might result in a case where the actual written value keeps getting pushed off and off forever, and we never actually write to the server.
+                    commitQueue[i].canBeClobbered = false;
+                    status = "cancel";
+                    if (Querysub.DEBUG_CALLS) {
+                        console.log(`[Querysub] ${blue("removed redundant call")}      ${green(debugCallSpec(commitQueue[i].callSpec))}  has all of it's writes handled by ${green(debugCallSpec(commitQueue[index].callSpec))}`);
+                    }
+                    break;
+                }
+            }
+        }
+    }
+
+    // Resolve all the values before us as well, before even our own promise, and then wait on that promise. That way, everything finishes in the same order it starts. 
+    if (index >= 0) {
+        for (let i = 0; i <= index; i++) {
+            commitQueue[i].resolveOrderedPromise.resolve();
+        }
+        commitQueue.splice(0, index + 1);
+
+        Querysub.localCommit(() => commitQueueCount().count -= index + 1);
+    }
+    await obj.resolveOrderedPromise.promise;
+
+
+    return status;
+}
+
+export async function flushPredictionQueueBase() {
+    let promises: Promise<void>[] = [];
+    for (let commitQueue of commitQueues.values()) {
+        for (let obj of commitQueue) {
+            obj.commitTime.resolve();
+            promises.push(obj.resolveOrderedPromise.promise);
+        }
+    }
+    await Promise.all(promises);
+}
+export function syncHasPendingPredictionsBase(): boolean {
+    return commitQueueCount().count > 0;
+}