querysub 0.437.0 → 0.439.0

package/src/3-path-functions/PathFunctionRunner.ts

@@ -9,7 +9,7 @@ import { getPathFromStr, getPathIndex } from "../path";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { getProxyPath } from "../2-proxy/pathValueProxy";
 import { atomicObjectRead, atomicObjectWrite, doProxyOptions, isProxyBlockedByOrder, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { authorityStorage, compareTime, debugTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { authorityStorage, compareTime, debugTime, MAX_ACCEPTED_CHANGE_AGE, parseDebugTime, PathValue, Time } from "../0-path-value-core/pathValueCore";
 import { getModuleFromSpec } from "./pathFunctionLoader";
 import debugbreak from "debugbreak";
 import { parseArgs } from "./PathFunctionHelpers";
@@ -164,6 +164,15 @@ function getDebugName(call: CallSpec, functionSpec: FunctionSpec | undefined, co
         return `${mainPart}`;
     }
 }
+export function parseDebugName(debugName: string) {
+    let parts = debugName.split("|");
+    if (parts.length !== 3) return undefined;
+    return {
+        file: parts[0],
+        functionId: parts[1],
+        runAtTime: parseDebugTime(parts[2]),
+    };
+}
 
 type PermissionsCheckType = {
     new(callerMachineId: { callerMachineId: string; callerIP: string; }): {
@@ -531,7 +540,9 @@ export class PathFunctionRunner {
             return;
         }
 
-        console.info(`New function call: ${getDebugName(callSpec, functionSpec, true)}`, {
+        let debugNameColored = getDebugName(callSpec, functionSpec, true);
+        let debugName = getDebugName(callSpec, functionSpec);
+        console.info(`New function call: ${debugNameColored}`, {
             callId: callSpec.CallId,
             timeId: callSpec.runAtTime.time,
             functionId: callSpec.FunctionId,
@@ -564,13 +575,13 @@ export class PathFunctionRunner {
             }
             if (PathFunctionRunner.DEBUG_CALLS) {
                 let fraction = getRoutingOverridePart(callSpec.CallId)?.route;
-                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction?.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${getDebugName(callSpec, functionSpec, true)}`);
+                console.log(`${yellow("Function run fallback")} (primary servery failed to run function) after ${formatTime(secondaryDelay)}. Fraction ${fraction?.toFixed(3)}. Primary ${this.config.shardRange.startFraction.toFixed(3)} to ${this.config.shardRange.endFraction.toFixed(3)}. Secondary ${this.config.secondaryShardRange?.startFraction.toFixed(3)} to ${this.config.secondaryShardRange?.endFraction.toFixed(3)}. for ${debugNameColored}`);
             }
         }
 
         PathFunctionRunner.RUN_START_COUNT++;
         if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`STARTING ${getDebugName(callSpec, functionSpec, true)}`);
+            console.log(`STARTING ${debugNameColored}`);
         }
         let startTime = Date.now();
         let runCount = 0;
@@ -618,29 +629,29 @@ export class PathFunctionRunner {
                     devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
                 }
 
-                console.info(`Running function: ${getDebugName(callSpec, functionSpec, true)}`, {
+                console.info(`Running function: ${debugNameColored}`, {
                     callId: callSpec.CallId,
                     outerLoop: retries,
                 });
 
                 await proxyWatcher.commitFunction({
                     canWrite: true,
-                    debugName: getDebugName(callSpec, functionSpec),
+                    debugName: debugName,
+                    source: debugName,
                     runAtTime: callSpec.runAtTime,
                     getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callSpec)),
                     nestedCalls: "inline",
                     temporary: true,
                     maxLocksOverride: devFunctionMetadata ? devFunctionMetadata.maxLocksOverride : functionSpec.maxLocksOverride,
-                    source: callSpec.CallId,
                     watchFunction: function runCallWatcher() {
                         runCount++;
                         stats.lastInternalLoopCount = runCount;
                         stats.totalInternalLoopCount++;
                         if (PathFunctionRunner.DEBUG_CALLS) {
-                            console.log(`Evaluating (try count ${runCount}) ${getDebugName(callSpec, functionSpec, true)}`);
+                            console.log(`Evaluating (try count ${runCount}) ${debugNameColored}`);
                         }
                         if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
-                            let errorMessage = `MAX_WATCH_LOOPS exceeded for ${getDebugName(callSpec, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`;
+                            let errorMessage = `MAX_WATCH_LOOPS exceeded for ${debugNameColored}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`;
                             console.error(errorMessage, {
                                 callId: callSpec.CallId,
                                 runCount,
@@ -662,7 +673,7 @@ export class PathFunctionRunner {
                                 atomicObjectRead(syncedModule.Sources[callSpec.FunctionId])
                             );
                             if (!syncedSpec) {
-                                throw new Error(`Function spec not found for ${getDebugName(callSpec, functionSpec, true)}`);
+                                throw new Error(`Function spec not found for ${debugNameColored}`);
                             }
 
                             //  (We also need to depend on the RIGHT function spec).
@@ -775,7 +786,7 @@ export class PathFunctionRunner {
                 // }
                 //if (PathFunctionRunner.DEBUG_CALLS)
                 {
-                    console.log(`Function run error ${getDebugName(callSpec, functionSpec, true)}`, {
+                    console.log(`Function run error ${debugNameColored}`, {
                         callId: callSpec.CallId,
                         error: e.stack,
                     });
@@ -799,7 +810,7 @@ export class PathFunctionRunner {
         }
 
         if (PathFunctionRunner.DEBUG_CALLS) {
-            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${getDebugName(callSpec, functionSpec, true)}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
+            console.log(`FINISHED${nooped ? " (skipped)" : ""} ${debugNameColored}, writes: ${finalWrites?.length}, took ${blue(formatTime(Date.now() - startTime))}`);
         }
 
         let wallTime = Date.now() - startTime;
@@ -814,7 +825,7 @@ export class PathFunctionRunner {
             });
         }
 
-        console.info(`Finished function evaluation: ${getDebugName(callSpec, functionSpec, true)}`, {
+        console.info(`Finished function evaluation: ${debugNameColored}`, {
             callId: callSpec.CallId,
             outerLoop: retries,
             finalWrites: finalWrites?.length,

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -1,87 +1,87 @@
-// import { disableMeasurements } from "socket-function/src/profiling/measure";
-// disableMeasurements();
-
-import "../inject";
-import yargs from "yargs";
-let yargObj = yargs(process.argv)
-    .option("fncshard", { type: "string", default: "0-1", desc: "Shard range as start-end (e.g., '0-0.5' or '0.25-0.75')" })
-    .option("fncsecshard", { type: "string", default: "", desc: "Secondary shard range as start-end (e.g., '0-0.5'). Values in this range (not matched in fncshard) are delayed." })
-    .option("filter", { type: "string", default: "", desc: `Filter to only include handle specific function calls (fncshard is still applied). For example, "a&b|c", using regular boolean rules.` })
-    .argv
-    ;
-
-import { logErrors } from "../errors";
-import { PathFunctionRunner } from "./PathFunctionRunner";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { getThreadKeyCert } from "../-a-auth/certs";
-import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { timeInMinute } from "socket-function/src/misc";
-import { getDomain, isLocal, isPublic } from "../config";
-import { green, magenta } from "socket-function/src/formatting/logColors";
-import { parseFilterSelector } from "../misc/filterable";
-import path from "path";
-import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
-
-async function main() {
-    const { PermissionsCheck } = await import("../4-querysub/permissions");
-    const { Querysub } = await import("../4-querysub/QuerysubController");
-    Error.stackTraceLimit = 20;
-
-    //ActionsHistory.LOG_ACTION_HISTORY = "runner";
-
-    // ClientWatcher.DEBUG_READS = true;
-    //ClientWatcher.DEBUG_WRITES = true;
-    // ClientWatcher.DEBUG_TRIGGERS = "heavy";
-    // authorityStorage.DEBUG_UNWATCH = true;
-
-    Querysub.SEND_FULL_HISTORY_ON_INITIAL_SYNC = true;
-
-    PathFunctionRunner.DEBUG_CALLS = true;
-    // debugCoreMode();
-
-    await Querysub.hostService("PathFunctionRunnerMain");
-
-    // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
-    //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
-    //  (it should be on the same local network as the path value authorities).
-    ClientWatcher.WATCH_STICK_TIME = timeInMinute * 5;
-
-    let [shardStart, shardEnd] = yargObj.fncshard.split("-").map(v => parseFloat(v));
-    if (isNaN(shardStart) || isNaN(shardEnd)) {
-        throw new Error(`Invalid fncshard format: ${yargObj.fncshard}. Expected format like "0-0.5"`);
-    }
-
-    let secondaryShardRange: { startFraction: number; endFraction: number } | undefined;
-    if (yargObj.fncsecshard) {
-        const [secondaryStart, secondaryEnd] = yargObj.fncsecshard.split("-").map(v => parseFloat(v));
-        if (isNaN(secondaryStart) || isNaN(secondaryEnd)) {
-            throw new Error(`Invalid fncsecshard format: ${yargObj.fncsecshard}. Expected format like "0-0.5"`);
-        }
-        secondaryShardRange = { startFraction: secondaryStart, endFraction: secondaryEnd };
-        console.log(green(`Sharding from ${shardStart} to ${shardEnd}. Fallback sharding (dead function recovery) from ${secondaryStart} to ${secondaryEnd}`));
-    } else {
-        console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
-    }
-
-    let filterSelector = parseFilterSelector(yargObj.filter);
-
-    new PathFunctionRunner({
-        domainName: getDomain(),
-        shardRange: { startFraction: shardStart, endFraction: shardEnd },
-        secondaryShardRange,
-        // TODO: Maybe abstract this out even more, so anything can plug in permissions checks?
-        PermissionsChecker: PermissionsCheck,
-        filterSelector,
-    });
-
-    if (!isPublic()) {
-        // We have to import deploy, otherwise local moduleIds won't be known about
-        let deployPath = path.resolve("./deploy.ts");
-        await import(deployPath);
-    }
-
-    if (yargObj.filter) {
-        console.log(magenta(`Only running functions that match the filter: ${yargObj.filter}. Either use --fncfilter dev in your http server to can calls here, or use ?setfncfilter=${yargObj.filter} in the browser to trigger functions that match this filter.`));
-    }
-}
-logErrors(main());
+// import { disableMeasurements } from "socket-function/src/profiling/measure";
+// disableMeasurements();
+
+import "../inject";
+import yargs from "yargs";
+let yargObj = yargs(process.argv)
+    .option("fncshard", { type: "string", default: "0-1", desc: "Shard range as start-end (e.g., '0-0.5' or '0.25-0.75')" })
+    .option("fncsecshard", { type: "string", default: "", desc: "Secondary shard range as start-end (e.g., '0-0.5'). Values in this range (not matched in fncshard) are delayed." })
+    .option("filter", { type: "string", default: "", desc: `Filter to only include handle specific function calls (fncshard is still applied). For example, "a&b|c", using regular boolean rules.` })
+    .argv
+    ;
+
+import { logErrors } from "../errors";
+import { PathFunctionRunner } from "./PathFunctionRunner";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getThreadKeyCert } from "../-a-auth/certs";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { timeInMinute } from "socket-function/src/misc";
+import { getDomain, isLocal, isPublic } from "../config";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+import { parseFilterSelector } from "../misc/filterable";
+import path from "path";
+import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
+
+async function main() {
+    const { PermissionsCheck } = await import("../4-querysub/permissions");
+    const { Querysub } = await import("../4-querysub/QuerysubController");
+    Error.stackTraceLimit = 20;
+
+    //ActionsHistory.LOG_ACTION_HISTORY = "runner";
+
+    // ClientWatcher.DEBUG_READS = true;
+    //ClientWatcher.DEBUG_WRITES = true;
+    // ClientWatcher.DEBUG_TRIGGERS = "heavy";
+    // authorityStorage.DEBUG_UNWATCH = true;
+
+    Querysub.SEND_FULL_HISTORY_ON_INITIAL_SYNC = true;
+
+    PathFunctionRunner.DEBUG_CALLS = true;
+    // debugCoreMode();
+
+    await Querysub.hostService("PathFunctionRunnerMain");
+
+    // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
+    //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
+    //  (it should be on the same local network as the path value authorities).
+    ClientWatcher.WATCH_STICK_TIME = timeInMinute * 5;
+
+    let [shardStart, shardEnd] = yargObj.fncshard.split("-").map(v => parseFloat(v));
+    if (isNaN(shardStart) || isNaN(shardEnd)) {
+        throw new Error(`Invalid fncshard format: ${yargObj.fncshard}. Expected format like "0-0.5"`);
+    }
+
+    let secondaryShardRange: { startFraction: number; endFraction: number } | undefined;
+    if (yargObj.fncsecshard) {
+        const [secondaryStart, secondaryEnd] = yargObj.fncsecshard.split("-").map(v => parseFloat(v));
+        if (isNaN(secondaryStart) || isNaN(secondaryEnd)) {
+            throw new Error(`Invalid fncsecshard format: ${yargObj.fncsecshard}. Expected format like "0-0.5"`);
+        }
+        secondaryShardRange = { startFraction: secondaryStart, endFraction: secondaryEnd };
+        console.log(green(`Sharding from ${shardStart} to ${shardEnd}. Fallback sharding (dead function recovery) from ${secondaryStart} to ${secondaryEnd}`));
+    } else {
+        console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
+    }
+
+    let filterSelector = parseFilterSelector(yargObj.filter);
+
+    new PathFunctionRunner({
+        domainName: getDomain(),
+        shardRange: { startFraction: shardStart, endFraction: shardEnd },
+        secondaryShardRange,
+        // TODO: Maybe abstract this out even more, so anything can plug in permissions checks?
+        PermissionsChecker: PermissionsCheck,
+        filterSelector,
+    });
+
+    if (!isPublic()) {
+        // We have to import deploy, otherwise local moduleIds won't be known about
+        let deployPath = path.resolve("./deploy.ts");
+        await import(deployPath);
+    }
+
+    if (yargObj.filter) {
+        console.log(magenta(`Only running functions that match the filter: ${yargObj.filter}. Either use --fncfilter dev in your http server to can calls here, or use ?setfncfilter=${yargObj.filter} in the browser to trigger functions that match this filter.`));
+    }
+}
+logErrors(main());