querysub 0.152.0 → 0.154.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.152.0",
+  "version": "0.154.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "preact": "^10.11.3",
-    "socket-function": "^0.82.0",
+    "socket-function": "^0.87.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.6.3",
     "yaml": "^2.5.0",
@@ -36,12 +36,12 @@
   "scripts": {
     "server": "yarn typenode ./src/server.ts",
     "serversharded": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
-    "function": "yarn typenode ./src/3-path-functions/PathFunctionRunnerMain.ts --function --local D:/repos/shard/",
-    "deploy": "yarn typenode ./src/4-deploy/deployMain.ts --domain querysub.com --local D:/repos/shard/",
+    "function": "yarn typenode ./src/3-path-functions/PathFunctionRunnerMain.ts --function --local",
+    "deploy": "yarn typenode ./src/4-deploy/deployMain.ts --domain querysub.com --local",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-    "test": "yarn typenode ./src/test/test.tsx --local D:/repos/shard/",
-    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local D:/repos/shard/"
+    "test": "yarn typenode ./src/test/test.tsx --local",
+    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local"
   },
   "bin": {
     "querysub-deploy": "./bin/deploy.js",

package/src/-b-authorities/cloudflareHelpers.ts

@@ -1,12 +1,21 @@
 import { httpsRequest } from "socket-function/src/https";
 import { getArchives } from "../-a-archives/archives";
 import { lazy } from "socket-function/src/caching";
-
+import { getStorageDir, getSubFolder } from "../fs";
+import fs from "fs";
 export const keys = getArchives("keys");
 
 export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: string }> => {
     let credsJSON = await keys.get("cloudflare.json");
-    if (!credsJSON) throw new Error(`b2:/keys/cloudflare.json is missing. It should contain { "key": "your-key", "email": "your-email" }`);
+    if (!credsJSON) {
+        let localPath = getStorageDir() + "cloudflare.json";
+        if (fs.existsSync(localPath)) {
+            credsJSON = fs.readFileSync(localPath);
+            await keys.set("cloudflare.json", credsJSON);
+        } else {
+            throw new Error(`b2:/keys/cloudflare.json is missing. It should contain { "key": "your-key", "email": "your-email" }`);
+        }
+    }
     return JSON.parse(credsJSON.toString());
 });
 

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,7 +1,7 @@
 import { delay, runInfinitePoll } from "socket-function/src/batching";
 import { cache } from "socket-function/src/caching";
 import { blue, magenta, red, yellow } from "socket-function/src/formatting/logColors";
-import { sha256HashBuffer, timeInHour } from "socket-function/src/misc";
+import { sha256HashBuffer, timeInHour, timeInSecond } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getLowUint32, getShortNumber } from "../bits";
 import { registerDynamicResource } from "../diagnostics/trackResources";
@@ -15,14 +15,15 @@ import { __getRoutingHash, authorityStorage, compareTime, debugTime, epochTime,
 import { getModuleFromSpec, watchModuleHotreloads } from "./pathFunctionLoader";
 import debugbreak from "debugbreak";
 import { parseArgs } from "./PathFunctionHelpers";
-import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
+import { PERMISSIONS_FUNCTION_ID, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { isLocal } from "../config";
+import { getDomain, isLocal } from "../config";
+import { getGitRef, getGitURL } from "../4-deploy/git";
 
 export const functionSchema = rawSchema<{
     [domainName: string]: {
@@ -220,7 +221,6 @@ export class PathFunctionRunner {
             Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, fullFraction);
         });
 
-        let callsToCall = new Map<string, { call: CallSpec; functionSpec: FunctionSpec }>();
         let runningCalls = new Set<string>();
 
         let callIdsToUnwatch = new Set<{ filePath: string; callId: string; }>();
@@ -258,89 +258,106 @@ export class PathFunctionRunner {
                     watcher.pendingWatches.paths.delete(getProxyPath(() => data()[filePath].Results[callId]));
                 }
                 callIdsToUnwatch.clear();
-                for (let [filePath, moduleData] of Object.entries(data())) {
-                    watchModuleCalls(filePath);
+
+                let moduleIds = Object.keys(data());
+                if (isLocal()) {
+                    moduleIds.push(...getAllDevelopmentModulesIds());
+                    moduleIds = Array.from(new Set(moduleIds));
                 }
 
                 let triggerPaths = new Set<string>();
-                for (let [filePath, moduleData] of Object.entries(data())) {
-                    let callPath = getProxyPath(() => data()[filePath].Calls);
+                for (let moduleId of moduleIds) {
+                    watchModuleCalls(moduleId);
+
+                    let callPath = getProxyPath(() => data()[moduleId].Calls);
                     triggerPaths.add(callPath);
 
                     // ALSO, Result changes (result rejections), can trigger a run
-                    let resultPath = getProxyPath(() => data()[filePath].Results);
+                    let resultPath = getProxyPath(() => data()[moduleId].Results);
                     triggerPaths.add(resultPath);
                 }
 
-                let callPathLength = 0;
-                if (triggerPaths.size > 0) {
-                    callPathLength = getPathDepth(Array.from(triggerPaths)[0]);
-                }
+                let callPathLength = (
+                    1 // domain
+                    + 1 // PathFunctionRunner
+                    + 1 // module
+                    + 1 // Calls
+                    + 1 // callId
+                );
 
                 let newCalls: { call: CallSpec; functionSpec: FunctionSpec }[] = [];
 
                 if (watcher.triggeredByChanges) {
                     for (let path of watcher.triggeredByChanges.paths) {
-                        let callPathEquivalent = trimPathStrToDepth(path, callPathLength);
-                        if (triggerPaths.has(callPathEquivalent)) {
-                            const callId = getPathIndex(path, callPathLength);
-                            if (!callId) continue;
-                            let moduleId = getPathIndex(path, callPathLength - 2);
-                            if (!moduleId) continue;
-
-                            callsToCall.delete(callId);
-
-                            let moduleData = data()[moduleId];
-
-                            let result = atomicObjectRead(moduleData.Results[callId]);
-                            if (result) continue;
-
-                            let callData = atomicObjectRead(moduleData.Calls[callId]);
-                            if (!callData) continue;
-
-                            if (callData.DomainName !== domainName) continue;
-                            if (callData.ModuleId !== moduleId) continue;
-                            if (callData.CallId !== callId) continue;
-
-                            let functionSpec = atomicObjectRead(moduleData.Sources[callData.FunctionId]);
-                            if (!functionSpec) {
-                                if (isSynced(moduleData.Sources[callData.FunctionId])) {
-                                    console.warn(yellow(`Cannot call ${callData.DomainName}.${callData.ModuleId}.Sources.${callData.FunctionId} because it is not deployed`));
-                                }
+                        let callsOrResult = getPathIndex(path, 3);
+                        if (callsOrResult !== "Calls" && callsOrResult !== "Results") continue;
+                        if (getPathIndex(path, 1) !== "PathFunctionRunner") continue;
+                        if (getPathIndex(path, 0) !== domainName) continue;
+
+                        const callId = getPathIndex(path, 4);
+                        if (!callId) continue;
+                        let moduleId = getPathIndex(path, 2);
+                        if (!moduleId) continue;
+                        if (!moduleIds.includes(moduleId)) continue;
+
+                        let moduleData = data()[moduleId];
+
+                        let result = atomicObjectRead(moduleData.Results[callId]);
+                        if (result) continue;
+
+                        let callData = atomicObjectRead(moduleData.Calls[callId]);
+                        if (!callData) continue;
+
+                        // Ignore invalid data
+                        if (callData.DomainName !== domainName) continue;
+                        if (callData.ModuleId !== moduleId) continue;
+                        if (callData.CallId !== callId) continue;
+
+                        const functionId = callData.FunctionId;
+                        let functionSpec = atomicObjectRead(moduleData.Sources[functionId]);
+                        if (!functionSpec) {
+                            if (!isSynced(moduleData.Sources[callData.FunctionId])) {
                                 continue;
                             }
-
-                            // If we haven't synced the result, we can't know if it is ready or not
-                            //  (we'll run anyways once it is synced). Most other parts are immutable,
-                            //  but the result is by definition not, so... it is one of the values we check!
-                            if (!isSynced(moduleData.Results[callId])) continue;
-
-                            if (runningCalls.has(callId)) continue;
-                            runningCalls.add(callId);
-
-                            let limitCount = queueLimitCounts.get(callId) || 0;
-                            limitCount++;
-                            queueLimitCounts.set(callId, limitCount);
-                            // NOTE: Calls are event writes, so... they should just clean themselves up, after we ignore them for long enough.
-                            if (limitCount >= MAX_QUEUE_COUNT) {
-                                // Only error the first time, as we don't need need that many errors
-                                if (limitCount === MAX_QUEUE_COUNT) {
-                                    console.error(`Queue limit reached (${limitCount}) for ${getDebugName(callData, functionSpec, true)}`);
-                                }
+                            if (isLocal()) {
+                                functionSpec = getDevFunctionSpecFromCall(callData);
+                                if (!functionSpec) continue;
+                            } else {
+                                console.warn(yellow(`Cannot call ${callData.DomainName}.${callData.ModuleId}.Sources.${callData.FunctionId} because it is not deployed`));
                                 continue;
                             }
+                        }
 
-                            if (PathFunctionRunner.DEBUG_CALLS) {
-                                console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
-                                let resultsPath = getProxyPath(() => moduleData.Results[callId]);
-                                let history = authorityStorage.getValuePlusHistory(resultsPath);
-                                console.log(`    History: ${history.length}`);
-                                for (let { valid, time, canGCValue } of history) {
-                                    console.log(`        ${valid ? "✅" : "❌"} ${debugTime(time)} ${canGCValue ? "value is undefined" : ""}`);
-                                }
+                        // If we haven't synced the result, we can't know if it is ready or not
+                        //  (we'll run anyways once it is synced). Most other parts are immutable,
+                        //  but the result is by definition not, so... it is one of the values we check!
+                        if (!isSynced(moduleData.Results[callId])) continue;
+
+                        if (runningCalls.has(callId)) continue;
+                        runningCalls.add(callId);
+
+                        let limitCount = queueLimitCounts.get(callId) || 0;
+                        limitCount++;
+                        queueLimitCounts.set(callId, limitCount);
+                        // NOTE: Calls are event writes, so... they should just clean themselves up, after we ignore them for long enough.
+                        if (limitCount >= MAX_QUEUE_COUNT) {
+                            // Only error the first time, as we don't need need that many errors
+                            if (limitCount === MAX_QUEUE_COUNT) {
+                                console.error(`Queue limit reached (${limitCount}) for ${getDebugName(callData, functionSpec, true)}`);
+                            }
+                            continue;
+                        }
+
+                        if (PathFunctionRunner.DEBUG_CALLS) {
+                            console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
+                            let resultsPath = getProxyPath(() => moduleData.Results[callId]);
+                            let history = authorityStorage.getValuePlusHistory(resultsPath);
+                            console.log(`    History: ${history.length}`);
+                            for (let { valid, time, canGCValue } of history) {
+                                console.log(`        ${valid ? "✅" : "❌"} ${debugTime(time)} ${canGCValue ? "value is undefined" : ""}`);
                             }
-                            newCalls.push({ call: callData, functionSpec });
                         }
+                        newCalls.push({ call: callData, functionSpec });
                     }
                 }
 
@@ -368,6 +385,27 @@ export class PathFunctionRunner {
         });
         registerDynamicResource("paths|PathFunctionRunner.watches", () => watcher.lastWatches.paths.size);
         registerDynamicResource("paths|PathFunctionRunner.outstandingCalls", () => outstandingCalls);
+
+        // If local, periodically check all modules
+        //  NOTE: `yarn deploy` commits the moduleIds to the database, but calls do not, as this would require either an extra
+        //      read, or an extra write. So this is somewhat a hack, but also fairly safe.
+        if (isLocal()) {
+            let prevModuleIds = new Set<string>();
+            runInfinitePoll(timeInSecond * 15, () => {
+                let moduleIds = getAllDevelopmentModulesIds();
+                let changed = false;
+                for (let moduleId of moduleIds) {
+                    if (!prevModuleIds.has(moduleId)) {
+                        changed = true;
+                        break;
+                    }
+                }
+                prevModuleIds = new Set(moduleIds);
+                if (changed) {
+                    watcher.explicitlyTrigger();
+                }
+            });
+        }
     }
 
 
@@ -520,43 +558,48 @@ export class PathFunctionRunner {
                         let syncedModule = skipPermissions(() =>
                             functionSchema()[callPath.DomainName].PathFunctionRunner[callPath.ModuleId]
                         );
-                        let syncedSpec = skipPermissions(() =>
-                            atomicObjectRead(syncedModule.Sources[callPath.FunctionId])
-                        );
-                        if (!syncedSpec) {
-                            throw new Error(`Function spec not found for ${getDebugName(callPath, functionSpec, true)}`);
-                        }
 
-                        //  (We also need to depend on the RIGHT function spec).
-                        if (
-                            syncedSpec && (
-                                syncedSpec.DomainName !== functionSpec.DomainName
-                                || syncedSpec.ModuleId !== functionSpec.ModuleId
-                                || syncedSpec.FilePath !== functionSpec.FilePath
-                                || syncedSpec.FunctionId !== functionSpec.FunctionId
-                                || syncedSpec.exportPathStr !== functionSpec.exportPathStr
-                                || syncedSpec.FilePath !== functionSpec.FilePath
-                                || syncedSpec.gitRef !== functionSpec.gitRef
-                                || syncedSpec.gitURL !== functionSpec.gitURL
-                            )
-                        ) {
-                            isFunctionSpecOutdated = true;
-                            functionSpec = {
-                                DomainName: syncedSpec.DomainName,
-                                ModuleId: syncedSpec.ModuleId,
-                                FilePath: syncedSpec.FilePath,
-                                FunctionId: syncedSpec.FunctionId,
-                                exportPathStr: syncedSpec.exportPathStr,
-                                gitRef: syncedSpec.gitRef,
-                                gitURL: syncedSpec.gitURL,
-                            };
-                            return;
+                        // Make sure we are running the latest function (by checking it here, we also lock the values,
+                        //  so if it is updated we atomically rerun / reject all calls which were racing with us).
+                        if (!isLocal()) {
+                            let syncedSpec = skipPermissions(() =>
+                                atomicObjectRead(syncedModule.Sources[callPath.FunctionId])
+                            );
+                            if (!syncedSpec) {
+                                throw new Error(`Function spec not found for ${getDebugName(callPath, functionSpec, true)}`);
+                            }
+
+                            //  (We also need to depend on the RIGHT function spec).
+                            if (
+                                syncedSpec && (
+                                    syncedSpec.DomainName !== functionSpec.DomainName
+                                    || syncedSpec.ModuleId !== functionSpec.ModuleId
+                                    || syncedSpec.FilePath !== functionSpec.FilePath
+                                    || syncedSpec.FunctionId !== functionSpec.FunctionId
+                                    || syncedSpec.exportPathStr !== functionSpec.exportPathStr
+                                    || syncedSpec.FilePath !== functionSpec.FilePath
+                                    || syncedSpec.gitRef !== functionSpec.gitRef
+                                    || syncedSpec.gitURL !== functionSpec.gitURL
+                                )
+                            ) {
+                                isFunctionSpecOutdated = true;
+                                functionSpec = {
+                                    DomainName: syncedSpec.DomainName,
+                                    ModuleId: syncedSpec.ModuleId,
+                                    FilePath: syncedSpec.FilePath,
+                                    FunctionId: syncedSpec.FunctionId,
+                                    exportPathStr: syncedSpec.exportPathStr,
+                                    gitRef: syncedSpec.gitRef,
+                                    gitURL: syncedSpec.gitURL,
+                                };
+                                return;
+                            }
                         }
 
                         let evalTimeStart = Date.now();
                         try {
                             let args = parseArgs(callPath);
-                            overrideCurrentCall({ spec: callPath, fnc: syncedSpec, }, () => {
+                            overrideCurrentCall({ spec: callPath, fnc: functionSpec, }, () => {
                                 baseFunction(...args);
                             });
                         } finally {
@@ -671,6 +714,34 @@ export async function preloadFunctions(specs: FunctionSpec[]) {
     }));
 }
 
+export function getDevFunctionSpecFromCall(call: {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+}): FunctionSpec | undefined {
+    let domainName = call.DomainName;
+    let moduleId = call.ModuleId;
+    let functionId = call.FunctionId;
+
+    let devModule = getDevelopmentModule(moduleId);
+    if (!devModule) {
+        console.warn(yellow(`Cannot call ${domainName}.${moduleId}.Sources.${functionId} because the module is not deployed and not referenced in deploy.ts`));
+        return undefined;
+    }
+    let filePath = getModuleRelativePath(devModule);
+    let gitURL = getGitURL();
+    let gitRef = getGitRef();
+    return {
+        DomainName: domainName,
+        ModuleId: moduleId,
+        FunctionId: functionId,
+        exportPathStr: getExportPath(functionId),
+        FilePath: filePath,
+        gitURL,
+        gitRef,
+    };
+}
+
 class FunctionPreloaderBase {
     async preloadFunctions(specs: FunctionSpec[]) {
         if (isLocal()) return;

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -22,13 +22,13 @@ import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { PermissionsCheck } from "../4-querysub/permissions";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { timeInMinute } from "socket-function/src/misc";
-import { getDomain, isPublic } from "../config";
+import { devDebugbreak, getDomain, isLocal, isPublic } from "../config";
 import { publishMachineARecords } from "../-e-certs/EdgeCertController";
 import { green } from "socket-function/src/formatting/logColors";
 import { parseFilterSelector } from "../misc/filterable";
+import path from "path";
 
 async function main() {
-
     listenOnDebugger("FunctionRunner");
     Error.stackTraceLimit = 20;
 
@@ -68,5 +68,11 @@ async function main() {
         PermissionsChecker: PermissionsCheck,
         filterSelector,
     });
+
+    if (isLocal()) {
+        // We have to import deploy, otherwise local moduleIds won't be known about
+        let deployPath = path.resolve("./deploy.ts");
+        await import(deployPath);
+    }
 }
 logErrors(main());

package/src/3-path-functions/pathFunctionLoader.ts

@@ -63,14 +63,15 @@ const getLocalPathRemapping = lazy((): { [gitUrl: string]: string } => {
         let packageJSONObj = JSON.parse(packageJSON);
         let repo = packageJSONObj.repository as { type: string, url: string } | undefined;
         if (!repo) throw new Error(`No "repository" property in package.json at ${path}`);
-        if (repo.type !== "git") {
-            throw new Error(`Repository type ${JSON.stringify(repo.type)} is not supported yet. Only "git" is presently supported.`);
+        if (repo.type !== "git" && repo.type !== "git+https" && repo.type !== "https") {
+            throw new Error(`Repository type ${JSON.stringify(repo.type)} is not supported yet. Only "git", "git+https", and "https" are presently supported.`);
         }
         mapping[repo.url] = path;
-        // Also, use the git@ version
-        // Replace "https://*/" with "git@*:"
-        let gitAtUrl = repo.url.replace(/^https:\/\/([^\/]+)\//, "git@$1:");
-        mapping[gitAtUrl] = path;
+        // https://github.com/sliftist/qs-cyoa.git
+        // git@github.com:sliftist/qs-cyoa.git
+        mapping[repo.url.replace(/^https:\/\/([^\/]+)\//, "git@$1:")] = path;
+        // Go from git to https as well, as it might be https
+        mapping[repo.url.replace(/^git@([^:]+):(.+)$/, "https://$1/$2")] = path;
     }
     return mapping;
 });
@@ -131,6 +132,10 @@ let moduleResolver = async (spec: {
     gitURL: string;
     gitRef: string;
 }) => {
+    if (isLocal()) {
+        // Probably a bug. The local path should have matched.
+        devDebugbreak();
+    }
     let gitURL = spec.gitURL;
     let urlForPath = gitURL;
 

package/src/3-path-functions/syncSchema.ts

@@ -5,7 +5,7 @@ import { Args } from "socket-function/src/types";
 import { appendToPathStr, getPathFromStr, getPathStr, rootPathStr } from "../path";
 import { writeFunctionCall } from "./PathFunctionHelpers";
 import { CallSpec, functionSchema } from "./PathFunctionRunner";
-import { getDomain } from "../config";
+import { getDomain, isLocal } from "../config";
 import { isHotReloading } from "socket-function/hot/HotReloadController";
 import { Schema2, Schema2Fncs, Schema2T, SchemaPath } from "../2-proxy/schema2";
 import { PathValueProxyWatcher, atomic, proxyWatcher, registerSchema } from "../2-proxy/PathValueProxyWatcher";
@@ -399,9 +399,18 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
         // NOTE: We clobber the previous value, as we might just be hotreloading, which
         //  could mean the exports value has our previous value.
         module.exports[SCHEMA_EXPORT_KEY] = result;
+        developmentModules.set(moduleId, module);
         return result;
     };
 }
+let developmentModules = new Map<string, NodeJS.Module>();
+
+export function getDevelopmentModule(moduleId: string): NodeJS.Module | undefined {
+    return developmentModules.get(moduleId);
+}
+export function getAllDevelopmentModulesIds(): string[] {
+    return Array.from(developmentModules.keys());
+}
 
 function isChildPath(spec: SchemaPath, path: string[]): boolean {
     if (spec.length > path.length) return false;

package/src/4-deploy/edgeBootstrap.ts

@@ -35,7 +35,14 @@ export async function getEdgeBootstrapScript(config: {
 }): Promise<string> {
     return await measureBlock(async function getEdgeBootstrapScript() {
         let cachedConfig = await getCachedConfig(config.edgeNodeConfigURL);
-        return `(${edgeNodeFunction.toString()})(${JSON.stringify({ ...config, cachedConfig })});`;
+        let allowedArgs = ["--local"];
+        // TODO: Be smarter about this (getting values and not just flags)
+        let argv = process.argv.filter(x => allowedArgs.includes(x));
+        return `(${edgeNodeFunction.toString()})(${JSON.stringify({
+            ...config,
+            cachedConfig,
+            argv,
+        })});`;
     });
 }
 
@@ -64,7 +71,9 @@ declare global {
 async function edgeNodeFunction(config: {
     edgeNodeConfigURL: string;
     cachedConfig: EdgeNodesIndex | undefined;
+    argv: string[];
 }) {
+    process.argv = config.argv;
     // IMPORTANT! Everything in this function is embedded, so this function can't use any external imports
     //  (except for type imports, of course).
 

package/src/4-querysub/Querysub.ts

@@ -3,7 +3,7 @@
 //  a file prevents it from transforming it, so we'll miss a lot of files doing it this late).
 import "../inject";
 
-import { isNode, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { isNode, isNodeTrue, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
 
 import { SocketFunction } from "socket-function/SocketFunction";
 import { isHotReloading, onHotReload, watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadController";
@@ -17,7 +17,7 @@ import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWa
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { isValueProxy2, getProxyPath } from "../2-proxy/pathValueProxy";
-import { getCurrentCallAllowUndefined, getCurrentCall, CallSpec } from "../3-path-functions/PathFunctionRunner";
+import { getCurrentCallAllowUndefined, getCurrentCall, CallSpec, PathFunctionRunner } from "../3-path-functions/PathFunctionRunner";
 import { listenOnDebugger } from "../diagnostics/listenOnDebugger";
 import { logErrors } from "../errors";
 import { getLastPathPart, getPathIndexAssert, getPathStr2, hack_setPackedPathSuffix } from "../path";
@@ -47,9 +47,45 @@ import { hookErrors } from "../diagnostics/errorLogs/hookErrors";
 import { Schema2, Schema2T, t } from "../2-proxy/schema2";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
 import { isDynamicModule } from "../3-path-functions/pathFunctionLoader";
+import yargs from "yargs";
 
 export { t };
 
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("verboseall", { type: "boolean", desc: "Log all everything (except network)" })
+    .option("verbose", { type: "boolean", desc: "Log all writes and reads" })
+    .option("verbosewrites", { type: "boolean", desc: "Log all writes" })
+    .option("verbosereads", { type: "boolean", desc: "Log all reads" })
+    .option("verbosecalls", { type: "boolean", desc: "Log all calls" })
+    .option("verbosenetwork", { type: "boolean", desc: "Log all network activity" })
+    .argv || {}
+    ;
+setImmediate(() => {
+    if (yargObj.verbose) {
+        ClientWatcher.DEBUG_WRITES = true;
+        ClientWatcher.DEBUG_READS = true;
+    }
+    if (yargObj.verbosewrites) {
+        ClientWatcher.DEBUG_WRITES = true;
+    }
+    if (yargObj.verbosereads) {
+        ClientWatcher.DEBUG_READS = true;
+    }
+    if (yargObj.verbosecalls) {
+        PathFunctionRunner.DEBUG_CALLS = true;
+        Querysub.DEBUG_CALLS = true;
+    }
+    if (yargObj.verboseall) {
+        ClientWatcher.DEBUG_WRITES = true;
+        ClientWatcher.DEBUG_READS = true;
+        PathFunctionRunner.DEBUG_CALLS = true;
+        Querysub.DEBUG_CALLS = true;
+    }
+    if (yargObj.verbosenetwork) {
+        SocketFunction.logMessages = true;
+    }
+});
+
 if (!registerGetCompressNetwork) {
     devDebugbreak();
 }
@@ -302,6 +338,38 @@ export class Querysub {
         proxyWatcher.getTriggeredWatcher().onInnerDisposed.push(callback);
     }
 
+    /** A more powerful version of omCommitFinished, which even waits for call predictions (or tries to).
+     *      - Also see afterPredictionsSynced, which runs the callback in a write.
+     *  NOTE: IDEALLY, you would just call a series of synced functions. They will be run in order,
+     *      with all the synced data, and just work. However... if you are running very expensive
+     *      asynchronous operations, you might need to use an async workflow.
+     *      "afterPredictions" helps in async workflows, by calling your callback after all local function
+     *          predictions finish. This allows you to (somewhat) read back their writes. It isn't perfect,
+     *          but it is safer then Promise.resolve().then(callback), or any type of setTimeout. Usually
+     *          the callback will be fast, except for the first time, when code will needed by loaded to
+     *          run the predictive functions.
+     *  NOTE: Only waits for calls BEFORE the callback is called. This is useful, but also means
+     *      you can't call this at the start of your function (as nothing will have been called yet).
+     *  NOTE: This can also be used to prevent
+     */
+    public static afterPredictions(callback: () => Promise<void>) {
+        let calls = proxyWatcher.getTriggeredWatcher().pendingCalls.map(x => x.call);
+        Querysub.onCommitFinished(async () => {
+            await Promise.all(calls.map(x => Querysub.onCallPredict(x)));
+            logErrors(callback());
+        });
+    }
+
+    /** Solely for use to prevent local writes from occuring before predictions. We MIGHT make this a framework thing,
+     *      or just not bother with it (as after a prediction is run once the code will be loaded allowing it to always
+     *      run before the next frame).
+     */
+    public static afterPredictionsSynced(callback: () => void) {
+        Querysub.afterPredictions(async () => {
+            Querysub.serviceWriteDetached(callback);
+        });
+    }
+
     public static onCallPredict = (x: CallSpec | undefined) => onCallPredict(x);
     /** NOTE: USUALLY you don't have to wait for predictions, as functions will run in order anyways.
      *      BUT, if you run a synced function in the UI, then somewhere unrelated try to read values
@@ -889,6 +957,11 @@ setImmediate(() => {
     hookErrors();
 });
 
+setImmediate(async () => {
+    // Import, so it registers addStatPeriodic
+    await import("../5-diagnostics/nodeMetadata");
+});
+
 registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
 registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
 
@@ -897,5 +970,6 @@ registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
 import "../diagnostics/watchdog";
 import "../diagnostics/trackResources";
 import "../diagnostics/benchmark";
-import { getEdgeNodeConfigURL, registerEdgeNode } from "../4-deploy/edgeNodes"; import { getEdgeBootstrapScript } from "../4-deploy/edgeBootstrap";
+import { getEdgeNodeConfigURL, registerEdgeNode } from "../4-deploy/edgeNodes";
+import { getEdgeBootstrapScript } from "../4-deploy/edgeBootstrap";