querysub 0.7.0 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.7.0",
+    "version": "0.9.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
         "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
         "pako": "^2.1.0",
         "preact": "^10.11.3",
-        "socket-function": "^0.33.0",
+        "socket-function": "^0.34.0",
         "terser": "^5.31.0",
         "typesafecss": "^0.6.3",
         "yaml": "^2.5.0",

package/src/-a-auth/certs.ts

@@ -447,6 +447,14 @@ export function decodeNodeId(nodeId: string): NodeIdParts | undefined {
         return undefined;
     }
     let parts = locationObj.address.split(".");
+    if (nodeId.startsWith("127-0-0-1.") && parts.length === 3) {
+        return {
+            threadId: "",
+            machineId: parts.at(-3) || "",
+            domain: parts.slice(-2).join("."),
+            port: locationObj.port,
+        };
+    }
     if (parts.length < 4) {
         return undefined;
     }

package/src/-g-core-values/NodeCapabilities.ts

@@ -54,11 +54,11 @@ export async function getControllerNodeIdList(
 ): Promise<string[]> {
     let nodeIdsToTest = await getAllNodeIds();
     let passedNodeIds = new Set<string>();
-    for (let nodeId of nodeIdsToTest) {
+    await Promise.all(nodeIdsToTest.map(async nodeId => {
         if (await doesNodeExposeController(nodeId, controller)) {
             passedNodeIds.add(nodeId);
         }
-    }
+    }));
     return Array.from(passedNodeIds);
 }
 

package/src/0-path-value-core/PathValueCommitter.ts

@@ -71,7 +71,7 @@ class PathValueCommitter {
                     //  - Check the caller proxyWatcher startTime vs lastSyncTime, to see how long it took from start to sync
                     //      (assuming the caller is a commitFunction, and not just a general watcher. If it is a general
                     //      watcher it might have taken a really long time to run, in which case check the watchFunction).
-                    let message = `MAX_CHANGE_AGE_EXCEEDED! Cannot commit write, that is before the max age ${pathValue.time.time} < ${maxAge}. Acceping this write would result in changes in the past that wouldn't propagate correctly`;
+                    let message = `MAX_CHANGE_AGE EXCEEDED! Cannot commit write, that is before the max age ${pathValue.time.time} < ${maxAge}. Acceping this write would result in changes in the past that wouldn't propagate correctly`;
                     console.error(red(message));
                     debugbreak(2);
                     debugger;

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -15,22 +15,22 @@ import { ClientWatcher, WatchSpecData, clientWatcher } from "../1-path-client/pa
 import { createPathValueProxy, getProxyPath, isValueProxy, isValueProxy2 } from "./pathValueProxy";
 import { authorityStorage, compareTime, ReadLock, epochTime, getNextTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, Time, getCreatorId } from "../0-path-value-core/pathValueCore";
 import { runCodeWithDatabase, rawSchema } from "./pathDatabaseProxyBase";
-import { CallSpec, DEPTH_TO_DATA, MODULE_INDEX, getCurrentCall } from "../3-path-functions/PathFunctionRunner";
-import { interceptCalls, runCall } from "../3-path-functions/PathFunctionHelpers";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
 import debugbreak from "debugbreak";
-import { FunctionMetadata, getSchemaObject, inlineNestedCalls } from "../3-path-functions/syncSchema";
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
-import { isClient } from "../config2";
-import { isDeploy } from "../3-path-functions/deployCheck";
 import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/NodePathAuthorities";
-import { PermissionsCheck } from "../4-querysub/permissions";
 import { registerPeriodic } from "../diagnostics/periodic";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { Schema2, Schema2Fncs } from "./schema2";
 import { getDomain } from "../config";
-import { Querysub } from "../4-querysub/QuerysubController";
+
+import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
+import type { FunctionMetadata } from "../3-path-functions/syncSchema";
+
+import { DEPTH_TO_DATA, MODULE_INDEX, getCurrentCall, getCurrentCallObj } from "../3-path-functions/PathFunctionRunner";
+import { inlineNestedCalls } from "../3-path-functions/syncSchema";
+import { interceptCalls, runCall } from "../3-path-functions/PathFunctionHelpers";
 
 // TODO: Break this into two parts:
 //  1) Run and get accesses
@@ -334,6 +334,7 @@ export function doProxyOptions<T>(options: Partial<WatcherOptions<any>>, callbac
 //          the dependent call, and find the key exists again, and the only difference will
 //          be a bit of lag, and value flicker.
 export const specialObjectWriteValue = "_specialObjectWriteValue_16c4c3bb43f24111976a2681c972f6f4";
+export const specialObjectWriteSymbol = Symbol("specialObjectWriteSymbol");
 // Values that don't add another proxy layer
 export function isTransparentValue(value: unknown) {
     return (
@@ -360,11 +361,6 @@ export type PermissionsChecker = {
 };
 
 
-const specialObjectWriteDepth = lazy(() => {
-    if (isDeploy()) return MODULE_INDEX;
-    return DEPTH_TO_DATA;
-});
-
 class SyncWatcherTag { }
 
 export class PathValueProxyWatcher {
@@ -406,7 +402,7 @@ export class PathValueProxyWatcher {
             throw new Error(`Tried to read a non-local path in a "noLocks" watcher, ${watcher.debugName}, path ${pathStr}`);
         }
 
-        if (watcher.permissionsChecker && !PermissionsCheck.skippingChecks) {
+        if (watcher.permissionsChecker) {
             if (!watcher.permissionsChecker.checkPermissions(pathStr).allowed) {
                 if (
                     !watcher.hasAnyUnsyncedAccesses()
@@ -571,6 +567,10 @@ export class PathValueProxyWatcher {
                 }
             }
         }
+        if (value === specialObjectWriteSymbol) {
+            value = specialObjectWriteValue;
+            allowSpecial = true;
+        }
 
         // Escape specialObjectWriteValue-like strings
         //  - Check for .startsWith, so we change all values. If we just added when it was ===, then
@@ -644,7 +644,7 @@ export class PathValueProxyWatcher {
                 let parentPathStr = pathStr;
                 while (true) {
                     parentPathStr = getParentPathStr(parentPathStr);
-                    if (getPathDepth(parentPathStr) < specialObjectWriteDepth()) break;
+                    if (getPathDepth(parentPathStr) < DEPTH_TO_DATA) break;
                     // We don't need to check all parents, as if any value is set, all parents will
                     //  likely have their specialObjectWriteValue set, due to a previous run of this function!
                     if (watcher.pendingWrites.has(parentPathStr)) break;
@@ -871,13 +871,6 @@ export class PathValueProxyWatcher {
             let runLeeway = MAX_ACCEPTED_CHANGE_AGE * 0.8;
             let runCutoffTime = Date.now() - runLeeway;
             if (options.runAtTime && options.runAtTime.time < runCutoffTime) {
-                // let timeSinceStart = Date.now() - startTime;
-                // // If we JUST started, we are just clearing out old function calls, so don't break on the error
-                // //  (Run leeway time + 60s)
-                // if (timeSinceStart > 1000 * 60 + runLeeway) {
-                //     debugbreak(2);
-                //     debugger;
-                // }
                 let message = `MAX_CHANGE_AGE_EXCEEDED! Cannot run watcher at time ${options.runAtTime.time} because it is older than the cutOff time of ${runCutoffTime}. Writing this far in the past would break things, and might be rejected by other authorities due to being too old.`;
                 console.error(red(message));
                 // NOTE: We could also adjust the to be more recent, to allow it to be commited anyway,
@@ -1704,8 +1697,8 @@ export class PathValueProxyWatcher {
     }
 }
 
-// domainName => moduleId => schema
-let schemas = new Map<string, Map<string, Schema2>>();
+// gitHash => domainName => moduleId => schema
+let schemas = new Map<string, Map<string, Map<string, Schema2>>>();
 
 // NOTE: We hardcode knowledge of how module data is nested to handle schemas. This... isn't great,
 //  but it makes the code faster, and if we change how module data is nested it will break
@@ -1727,28 +1720,13 @@ function getBaseMatchingSchema(pathStr: string): {
     schema: Schema2;
     nestedPath: string[];
 } | undefined {
-    //todonext
-    //  So... the current call, need to register the hash used for the call. This needs to be
-    //      used to prefer that schema from that same hash.
-    //      - And if we ever do cross repo calls... we need to store multiple hashes in the call
-    //          (or cascade hashes, etc, but we need to know all the hashes, and then use them here!)
-    //todonext;
-    // Ah, it's not quite so easy. So... pathFunctionLoader knows the hash on load (or that it's local),
-    //      but this information is not maintained when we are making the call (writeFunctionCall). BUT,
-    //      when we are in the call (unless it's a test call), we just called getModuleFromConfig
-    //      so we aren't too far away from knowing the hash, and passing it on.
-    //      - I think... we should return the hash, and then request it in overrideCurrentCall?
-    //todonext;
-    // Permissions checks are annoying, but... they DO call getModuleFromConfig at some time.
-    // ONLY managementPages.tsx does not actually have a module.
-    //todonext;
-    // Oh, it looks like managementPages.tsx can't operate the way it does. Hmm...
+    let gitHash = getCurrentCallObj()?.fnc.gitRef || "ambient";
 
     if (_noAtomicSchema) return undefined;
     if (getPathIndex(pathStr, 3) !== "Data") return undefined;
     let domain = getPathIndex(pathStr, 0)!;
     let moduleId = getPathIndex(pathStr, 2)!;
-    let schema = schemas.get(domain)?.get(moduleId);
+    let schema = schemas.get(gitHash)?.get(domain)?.get(moduleId);
     if (!schema) return undefined;
     let nestedPathStr = getPathSuffix(pathStr, 4);
     let nestedPath = getPathFromStr(nestedPathStr);
@@ -1759,15 +1737,19 @@ export function registerSchema(config: {
     schema: Schema2;
     domainName: string;
     moduleId: string;
+    gitHash: string;
 }) {
-    //todonext
-    //  Require passing the module information here
+    const { schema, domainName, moduleId, gitHash } = config;
+    let gitSchemas = schemas.get(gitHash);
+    if (!gitSchemas) {
+        schemas.set(gitHash, gitSchemas = new Map());
+    }
 
-    let domainSchemas = schemas.get(config.domainName);
+    let domainSchemas = gitSchemas.get(domainName);
     if (!domainSchemas) {
-        schemas.set(config.domainName, domainSchemas = new Map());
+        gitSchemas.set(domainName, domainSchemas = new Map());
     }
-    domainSchemas.set(config.moduleId, config.schema);
+    domainSchemas.set(moduleId, schema);
 }
 
 export let schemaPrefixes: { len: number; prefixes: Map<string, Schema2> }[] = [];
@@ -1780,6 +1762,11 @@ export function registerSchemaPrefix(config: {
     if (!list) {
         schemaPrefixes.push(list = { len, prefixes: new Map() });
     }
+    if (isNode()) {
+        if (list.prefixes.has(config.prefixPathStr)) {
+            throw new Error(`Prefix matches only work due to only having one version of the code ever load (except for during hot reloading development clientside). If we try to render multiple versions of the same code serverside (ex, because we pre-loaded new code during a deploy), it will break, as we can't distinguish them. ADD GIT HASH TO PREFIX SCHEMAS TO FIX THIS!`);
+        }
+    }
     list.prefixes.set(config.prefixPathStr, config.schema);
 }
 export function unregisterSchemaPrefix(config: {

package/src/3-path-functions/PathFunctionHelpers.ts

@@ -1,5 +1,5 @@
 import { getOwnMachineId } from "../-a-auth/certs";
-import { atomicObjectWrite, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { atomicObjectWrite, proxyWatcher, specialObjectWriteSymbol } from "../2-proxy/PathValueProxyWatcher";
 import { pathValueCommitter } from "../0-path-value-core/PathValueController";
 import { getNextTime } from "../0-path-value-core/pathValueCore";
 import { FunctionSpec, functionSchema, CallSpec } from "./PathFunctionRunner";
@@ -10,25 +10,82 @@ import cborx from "cbor-x";
 import { lazy } from "socket-function/src/caching";
 import { secureRandom } from "../misc/random";
 import { onCallPredict } from "../4-querysub/QuerysubController";
+import { isDefined } from "../misc";
+import { blue, green, red } from "socket-function/src/formatting/logColors";
+import { getPathStr2 } from "../path";
+import { sort } from "socket-function/src/misc";
 const cborxInstance = lazy(() => new cborx.Encoder({ structuredClone: true }));
 
-export async function deployFunction(spec: FunctionSpec) {
-    const { DomainName, ModuleId, FunctionId } = spec;
-    await proxyWatcher.commitFunction({
-        canWrite: true,
-        watchFunction: function writeTest() {
-            functionSchema()[DomainName].PathFunctionRunner[ModuleId].Sources[FunctionId] = atomicObjectWrite(spec);
-        },
-    });
-}
-export async function undeployFunction(spec: FunctionSpec) {
-    const { DomainName, ModuleId, FunctionId } = spec;
+
+// NOTE: We could deploy single functions, but... we will almost always be updating all functions at
+//      once, because keeping everything on the same git hash reduces a lot of potential bugs.
+export async function replaceFunctions(config: {
+    domainName: string;
+    functions: FunctionSpec[];
+}) {
     await proxyWatcher.commitFunction({
-        canWrite: true,
-        watchFunction: function writeTest() {
-            functionSchema()[DomainName].PathFunctionRunner[ModuleId].Sources[FunctionId] = undefined;
-        },
+        debugName: "replaceFunctions",
+        watchFunction() {
+            function debugFunction(func: FunctionSpec) {
+                return `${func.DomainName}:${func.FilePath}:${func.FunctionId}`;
+            }
+
+            let { domainName, functions } = config;
+            for (let fnc of functions) {
+                if (fnc.DomainName !== domainName) {
+                    throw new Error(`Tried to deploy function ${fnc.FunctionId} is not in domain ${domainName}, was in ${fnc.DomainName}`);
+                }
+            }
+
+            let base = functionSchema()[domainName].PathFunctionRunner;
+            let previousFunctions = Object.values(base).flatMap(x => Object.values(x.Sources)).filter(isDefined);
+
+            function getFunctionName(func: FunctionSpec) {
+                return getPathStr2(func.ModuleId, func.FunctionId);
+            }
+
+            let currentFunctions = functions;
+
+            let previousHashes = new Set(previousFunctions.map(x => JSON.stringify(x)));
+            let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
+
+            let prevFunctionNames = new Set(previousFunctions.map(getFunctionName));
+
+            let messages: {
+                text: string;
+                order: number;
+            }[] = [];
+
+            for (let previous of previousFunctions) {
+                let hash = JSON.stringify(previous);
+                if (currentHashes.has(hash)) continue;
+                messages.push({ text: `Removing  ${red(debugFunction(previous))}`, order: 3 });
+                base[previous.ModuleId].Sources[previous.FunctionId] = undefined;
+            }
+
+            console.log();
+            for (let func of currentFunctions) {
+                if (previousHashes.has(JSON.stringify(func))) {
+                    messages.push({ text: `Unchanged\t ${debugFunction(func)}`, order: 0 });
+                    continue;
+                }
+                if (!prevFunctionNames.has(getFunctionName(func))) {
+                    messages.push({ text: `Deploying new\t ${green(debugFunction(func))}`, order: 2 });
+                } else {
+                    messages.push({ text: `Deploying\t ${blue(debugFunction(func))}`, order: 1 });
+                }
+                base[func.ModuleId] = specialObjectWriteSymbol as any;
+                base[func.ModuleId].Sources[func.FunctionId] = atomicObjectWrite(func);
+            }
+
+            sort(messages, x => -x.order);
+            for (let message of messages) {
+                console.log(message.text);
+            }
+        }
     });
+
+    await pathValueCommitter.waitForValuesToCommit();
 }
 
 export const writeCall = {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -14,7 +14,6 @@ import { atomic, atomicObjectRead, atomicObjectWrite, doAtomicWrites, doProxyOpt
 import { __getRoutingHash, authorityStorage, compareTime, debugTime, epochTime, getNextTime, isCoreQuiet, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, Time } from "../0-path-value-core/pathValueCore";
 import { getModuleFromSpec, watchModuleHotreloads } from "./pathFunctionLoader";
 import debugbreak from "debugbreak";
-import { Querysub } from "../4-querysub/Querysub";
 import { parseArgs } from "./PathFunctionHelpers";
 import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -52,6 +51,7 @@ export type FunctionSpec = {
     exportPathStr: string;
     FilePath: string;
     gitURL: string;
+    /** Ex, the hash */
     gitRef: string;
 };
 
@@ -79,21 +79,31 @@ export type FunctionResult = ({
     runCount: number;
 };
 
-let currentCallSpec: CallSpec | undefined;
+let currentCallSpec: { spec: CallSpec; fnc: FunctionSpec; } | undefined;
 export function getCurrentCall() {
     if (!currentCallSpec) {
         debugbreak(2);
         debugger;
         throw new Error("Not presently in a call, and so cannot get call");
     }
-    return currentCallSpec;
+    return currentCallSpec.spec;
 }
 export function getCurrentCallAllowUndefined() {
+    return currentCallSpec?.spec;
+}
+export function getCurrentCallObj() {
     return currentCallSpec;
 }
-export function overrideCurrentCall<T>(call: CallSpec, code: () => T) {
+/** NOTE: We require the FunctionSpec so we know what has is loaded, to identify the
+ *      correct schema. This should be available, but if it isn't... we MIGHT be able
+ *      to work around it, if we can be assured the schema won't be used?
+ *      - This is so pre-loading of functions (which might happen without a deploy, if the deploy script
+ *          is killed part of the way through), doesn't change schemas until the deploy actually
+ *          updates the underlying functions.
+  */
+export function overrideCurrentCall<T>(config: { spec: CallSpec; fnc: FunctionSpec; }, code: () => T) {
     let prev = currentCallSpec;
-    currentCallSpec = call;
+    currentCallSpec = config;
     try {
         return code();
     } finally {
@@ -155,6 +165,7 @@ export class PathFunctionRunner {
 
     // Calls runCall on any calls that match our domain + shard range
     private async startWatching() {
+        const { Querysub } = await import("../4-querysub/Querysub");
         let { shardRange, secondaryShardRange } = this.config;
 
         // We will use PathValueProxyWatcher to watch the paths
@@ -398,6 +409,8 @@ export class PathFunctionRunner {
     }
 
     private async runCall(callPath: CallSpec, functionSpec: FunctionSpec): Promise<void> {
+        const { Querysub } = await import("../4-querysub/Querysub");
+
         const PermissionsChecker = this.config.PermissionsChecker;
         let skipPermissions = <T>(code: () => T) => code();
         if (PermissionsChecker) {
@@ -497,6 +510,9 @@ export class PathFunctionRunner {
                         let syncedSpec = skipPermissions(() =>
                             atomicObjectRead(syncedModule.Sources[callPath.FunctionId])
                         );
+                        if (!syncedSpec) {
+                            throw new Error(`Function spec not found for ${getDebugName(callPath, functionSpec, true)}`);
+                        }
 
                         //  (We also need to depend on the RIGHT function spec).
                         if (
@@ -526,7 +542,7 @@ export class PathFunctionRunner {
 
                         let evalTimeStart = Date.now();
                         let args = parseArgs(callPath);
-                        overrideCurrentCall(callPath, () => {
+                        overrideCurrentCall({ spec: callPath, fnc: syncedSpec, }, () => {
                             baseFunction(...args);
                         });
                         evalTime += Date.now() - evalTimeStart;

package/src/3-path-functions/deployMain.ts

@@ -1,24 +1,16 @@
 import "../inject";
 
-import { errorToUndefined } from "../errors";
-import { functionSchema, FunctionSpec } from "./PathFunctionRunner";
+import { FunctionSpec } from "./PathFunctionRunner";
 import path from "path";
 import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "./syncSchema";
 import { getModuleRelativePath } from "./syncSchema";
 import * as child_process from "child_process";
-import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { green, red } from "socket-function/src/formatting/logColors";
-import { isDefined, isEmpty } from "../misc";
 import fs from "fs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "../-a-auth/certs";
-import { deployFunction, undeployFunction } from "./PathFunctionHelpers";
-import { pathValueCommitter } from "../0-path-value-core/PathValueController";
+import { replaceFunctions } from "./PathFunctionHelpers";
 import { getPathStr2 } from "../path";
-import { cacheJSONArgsEqual } from "socket-function/src/caching";
 import { setIsDeploy } from "./deployCheck";
-import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import debugbreak from "debugbreak";
 
 import yargs from "yargs";
 import { isPublic } from "../config";
@@ -51,9 +43,6 @@ export async function deployMain() {
     const srcRoot = path.resolve(__dirname + "/../").replaceAll("\\", "/");
 
     let currentFunctions: FunctionSpec[] = [];
-    function debugFunction(func: FunctionSpec) {
-        return `${func.DomainName}:${func.FilePath}:${func.FunctionId}`;
-    }
 
     let gitDir = folderRoot;
     if (!fs.existsSync(gitDir + "/.git")) {
@@ -118,38 +107,7 @@ export async function deployMain() {
         }
     }
 
-    let previousFunctions = await proxyWatcher.commitFunction({
-        debugName: "readPreviousFunctions",
-        watchFunction(): FunctionSpec[] {
-            let base = functionSchema()[domainName].PathFunctionRunner;
-            return Object.values(base).flatMap(x => Object.values(x.Sources)).filter(isDefined);
-        },
-    });
-
-    let previousHashes = new Set(previousFunctions.map(x => JSON.stringify(x)));
-    let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
-    // BUG: This is incorrect, and breaks when you accidentally reuse a modelId
-    for (let func of currentFunctions) {
-        if (previousHashes.has(JSON.stringify(func))) {
-            console.log(`Unchanged ${debugFunction(func)}`);
-        }
-    }
-
-    for (let previous of previousFunctions) {
-        let hash = JSON.stringify(previous);
-        if (currentHashes.has(hash)) continue;
-        console.log(`Removing  ${red(debugFunction(previous))}`);
-        await undeployFunction(previous);
-    }
-
-    console.log();
-    for (let func of currentFunctions) {
-        if (previousHashes.has(JSON.stringify(func))) continue;
-        console.log(`Deploying ${green(debugFunction(func))}`);
-        await deployFunction(func);
-    }
-
-    await pathValueCommitter.waitForValuesToCommit();
+    await replaceFunctions({ domainName, functions: currentFunctions, });
 }
 function getGitURL(gitDir: string) {
     if (yargObj.nogit) return "git@github.com:nogit/nogit.git";

package/src/3-path-functions/pathFunctionLoader.ts

@@ -158,10 +158,29 @@ let moduleResolver = async (spec: FunctionSpec) => {
 };
 
 export function isDynamicModule(module: NodeJS.Module): boolean {
-    // Mostly a hack, but... it SHOULD be fine
-    let parts = module.filename.replaceAll("\\", "/").split("/");
+    return isDynamicModulePath(module.filename);
+}
+export function isDynamicModulePath(path: string): boolean {
+    let parts = path.replaceAll("\\", "/").split("/");
     return parts.includes("node_modules") || parts.includes("synced_repos");
 }
+/** Annoying, and slower than isDynamicModule, but... necessary. We can't expect the user to call isDynamicModule,
+ *      and our functions will all resolve to the root on purpose, so... we need to check the callstack!
+ */
+export function isCallerDynamicModule(): boolean {
+    return getCallstackFiles().some(isDynamicModulePath);
+}
+function getCallstackFiles(): string[] {
+    let stack = new Error().stack;
+    if (!stack) return [];
+    let lines = stack.split("\n");
+    let files = lines.map(line => {
+        let match = line.match(/\(([^)]+)\)/);
+        if (!match) return "";
+        return match[1];
+    });
+    return files;
+}
 
 async function getModuleFromSpecBase(
     spec: FunctionSpec
@@ -188,7 +207,18 @@ async function getModuleFromSpecBase(
         path = packagePath + specFilePath;
         deployPath = packagePath + "deploy.ts";
     }
+
     console.log(blue(`require(${JSON.stringify(path)})`));
+
+    // Set functionSpec for the next synchronous evaluation
+    //  - This breaks with async require, but... that's mostly just
+    //      clientside, so this SHOULD be fine? Maybe... Hopefully...
+    evaluatingFunctionSpec = spec;
+    void Promise.resolve().finally(() => {
+        if (evaluatingFunctionSpec === spec) {
+            evaluatingFunctionSpec = undefined;
+        }
+    });
     try {
         await SocketFunction.ignoreExposeCalls(async () => {
             // Import deploy, which should always exist, and provides a consistent
@@ -201,6 +231,10 @@ async function getModuleFromSpecBase(
         });
     } catch (e: any) {
         throw new Error(`Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
+    } finally {
+        if (evaluatingFunctionSpec === spec) {
+            evaluatingFunctionSpec = undefined;
+        }
     }
     let moduleId = require.resolve(path) || path;
     let module = require.cache[moduleId];
@@ -210,11 +244,7 @@ async function getModuleFromSpecBase(
         throw new Error(`Module not found: ${moduleId} (for ${spec.FunctionId})`);
     }
 
-    //todonext
-    //  Expose a function that goes from NodeJS.Module to FunctionSpec,
-    //      via a lookup we control?
-    //      - Hmm... we COULD change the module. This is nice, as it is more discoverable.
-    //          But... eh... I don't know. Just a lookup is safer...
+    moduleToSpec.set(module, spec);
 
     if (hotReloadPackagePath) {
         hotReloadUnderPath(hotReloadPackagePath);
@@ -223,6 +253,13 @@ async function getModuleFromSpecBase(
     return module;
 }
 
+let evaluatingFunctionSpec: FunctionSpec | undefined;
+
+let moduleToSpec = new Map<NodeJS.Module, FunctionSpec>();
+export function getSpecFromModule(module: NodeJS.Module): FunctionSpec | undefined {
+    return evaluatingFunctionSpec ?? moduleToSpec.get(module);
+}
+
 // Hot reload at or under the path
 const hotReloadUnderPath = cache((path: string) => {
     console.log(magenta(`Hot reloading under path: ${path}`));

package/src/3-path-functions/syncSchema.ts

@@ -18,6 +18,8 @@ import { CALL_PERMISSIONS_KEY } from "../4-querysub/permissionsShared";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
 import path from "path";
 import { isEmpty } from "../misc";
+import { getSpecFromModule } from "./pathFunctionLoader";
+import { isNode } from "typesafecss";
 
 // This is the the function id which should be used when creating the FunctionSpec (in order to load the module),
 //  to access the permissions in the schema.
@@ -265,7 +267,11 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
         }[] = [];
         if (schema) {
             let dataPrefixPath = getPathFromStr(getProxyPath(data));
-            registerSchema({ domainName, moduleId, schema });
+            let fnc = getSpecFromModule(module);
+            // NOTE: We call syncSchema a lot, so we default the git hash. However, for modules that provide
+            //  functions we call as synced functions, we SHOULD have a real hash here.
+            let gitHash = fnc?.gitRef || "ambient";
+            registerSchema({ domainName, moduleId, schema, gitHash });
 
             let gcDefs = Schema2Fncs.getGCObjects(schema);
             for (let { path, gcDelay } of gcDefs) {

package/src/4-dom/qreact.tsx

@@ -1,7 +1,6 @@
 import type preact from "preact";
 import { isNode, sort } from "socket-function/src/misc";
 import { isDeploy } from "../3-path-functions/deployCheck";
-import { Querysub } from "../4-querysub/QuerysubController";
 import { PermissionsCheck } from "../4-querysub/permissions";
 import { ClientWatcher, clientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
@@ -2319,4 +2318,8 @@ if (isClient()) {
     });
 }
 
-// #endregion
+// #endregion
+
+// NOTE: Import Querysub at the end, so we can export qreact before we require it. That way Querysub
+//      can statically access qreact.
+import { Querysub } from "../4-querysub/Querysub";

package/src/4-querysub/QuerysubController.ts

@@ -36,6 +36,7 @@ import * as prediction from "./querysubPrediction";
 import { getCallResultPath } from "./querysubPrediction";
 import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
 import { diskLog } from "../diagnostics/logs/diskLogger";
+import { assertIsManagementUser } from "../diagnostics/managementPages";
 setFlag(require, "preact", "allowclient", true);
 
 export { Querysub, id };
@@ -489,6 +490,7 @@ class QuerysubControllerBase {
         }
         pathWatcher.unwatchPath({ callback: callerId, ...config });
     }
+
     public async addCall(call: CallSpec) {
         if (Querysub.DEBUG_CALLS) {
             console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);

package/src/4-querysub/permissions.ts

@@ -1,4 +1,4 @@
-import { cache, cacheLimited } from "socket-function/src/caching";
+import { cache, cacheArgsEqual, cacheLimited } from "socket-function/src/caching";
 import { measureFnc, measureWrap, nameFunction } from "socket-function/src/profiling/measure";
 import { getPathSuffix, getPathDepth, trimPathStrToDepth, getPathFromStr, rootPathStr, getPathIndex, getPathStr1, joinPathStres, appendToPathStr, getPathStr } from "../path";
 import { atomic, atomicObjectRead, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
@@ -34,7 +34,7 @@ const callPermissionsPath = getPathStr1(CALL_PERMISSIONS_KEY);
  */
 export class PermissionsCheck {
     private dead = false;
-    public static skippingChecks = false;
+    private static skippingChecks = false;
 
     public static DEBUG = false;
 
@@ -42,8 +42,12 @@ export class PermissionsCheck {
         setImmediate(() => this.dead = true);
     }
 
-    private perSchema = cache((schema: SchemaObject) => {
-        return new PermissionsCheckSchema({ machineID: this.callerConfig.callerMachineId, IP: this.callerConfig.callerIP }, schema);
+    private perSchema = cacheArgsEqual((schema: SchemaObject, fnc: FunctionSpec) => {
+        return new PermissionsCheckSchema(
+            { machineID: this.callerConfig.callerMachineId, IP: this.callerConfig.callerIP },
+            schema,
+            fnc
+        );
     });
 
     private getModulePermissions = cache((domainName: string) => {
@@ -67,7 +71,12 @@ export class PermissionsCheck {
             }
 
             if (!module) return undefined;
-            return getSchemaObject(module);
+            let schema = getSchemaObject(module);
+            if (!schema) return undefined;
+            return {
+                schema,
+                fnc: functionConfig,
+            };
         });
     });
 
@@ -106,10 +115,10 @@ export class PermissionsCheck {
         if (!pathParts.moduleId) return { permissionsPath: trimPathStrToDepth(path, MODULE_INDEX + 1), allowed: false };
         if (!pathParts.rootKey) return { permissionsPath: trimPathStrToDepth(path, DEPTH_TO_DATA), allowed: false };
 
-        const schema = this.getModulePermissions(pathParts.domainName)(pathParts.moduleId);
-        if (!schema) return { permissionsPath: trimPathStrToDepth(path, MODULE_INDEX + 1), allowed: false };
+        const schemaObj = this.getModulePermissions(pathParts.domainName)(pathParts.moduleId);
+        if (!schemaObj) return { permissionsPath: trimPathStrToDepth(path, MODULE_INDEX + 1), allowed: false };
 
-        let instance = this.perSchema(schema);
+        let instance = this.perSchema(schemaObj.schema, schemaObj.fnc);
         instance.pathPrefix = pathPrefix;
 
         const rootKey = pathParts.rootKey;
@@ -151,7 +160,11 @@ export class PermissionsCheck {
 class PermissionsCheckSchema {
     public pathPrefix = rootPathStr;
 
-    constructor(private callerConfig: { machineID: string; IP: string }, private schema: SchemaObject) { }
+    constructor(
+        private callerConfig: { machineID: string; IP: string },
+        private schema: SchemaObject,
+        private fnc: FunctionSpec,
+    ) { }
     private exampleCall: CallSpec = {
         callerMachineId: this.callerConfig.machineID,
         callerIP: this.callerConfig.IP,
@@ -195,7 +208,10 @@ class PermissionsCheckSchema {
     //  so it is important to cache it based on the permissionsPath.
     private checkCache = cacheLimited(1000 * 1000, measureWrap((permissionsPath: string) => {
         let checks = this.getChecks(permissionsPath);
-        let allowed = overrideCurrentCall(this.exampleCall, () => {
+        let allowed = overrideCurrentCall({
+            spec: this.exampleCall,
+            fnc: this.fnc,
+        }, () => {
             // NOTE: If we read inside of a permissions check we don't want to ALSO check
             //  permissions checks on those because:
             //  1) It might infinitely loop

package/src/4-querysub/querysubPrediction.ts

@@ -475,7 +475,7 @@ export async function getCallWrites(config: {
     overrides?: PathValue[];
 }) {
     let { call, debugName } = config;
-    let { functionSpec } = await proxyWatcher.commitFunction({
+    const { functionSpec } = await proxyWatcher.commitFunction({
         watchFunction: function getModuleConfig() {
             //let moduleConfig = atomicObjectRead(functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Module);
             // if (!(call.DomainName in functionSchema())) {
@@ -516,7 +516,7 @@ export async function getCallWrites(config: {
         overrides: config.overrides,
         nestedCalls: "inline",
         watchFunction() {
-            return overrideCurrentCall(call, () => {
+            return overrideCurrentCall({ spec: call, fnc: functionSpec }, () => {
                 let args = parseArgs(call);
                 return baseFunction(...args);
             });

package/src/5-diagnostics/qreactDebug.tsx

@@ -16,7 +16,7 @@ import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { InputLabel, InputLabelURL } from "../library-components/InputLabel";
 import { URLParam } from "../library-components/URLParam";
 import { hotReloadingGuard, isHotReloading, onHotReload } from "socket-function/hot/HotReloadController";
-import { isDynamicModule } from "../3-path-functions/pathFunctionLoader";
+import { isCallerDynamicModule, isDynamicModule } from "../3-path-functions/pathFunctionLoader";
 
 // Map, so hot reloading doesn't break things
 let componentButtons = new Map<string, { title: string, callback: (component: ExternalRenderClass) => void }>();
@@ -25,7 +25,7 @@ export function addComponentButton(config: {
     title: string;
     callback: (component: ExternalRenderClass) => void;
 }) {
-    if (isDynamicModule(module)) return;
+    if (isCallerDynamicModule()) return;
     if (!isHotReloading() && componentButtons.has(config.title)) {
         throw new Error(`Component button with title ${config.title} already exists`);
     }

package/src/diagnostics/logs/diskLogger.ts

@@ -139,50 +139,54 @@ export function parseLogBuffer(buffer: Buffer): LogObj[] {
 }
 
 function safeCopyObject<T>(obj: T): T {
-    const maxFields = 20;
-    let fieldCount = 0;
-    const seen = new WeakSet();
-
-    function copy(value: unknown): unknown {
-        // Handle primitives
-        if (!canHaveChildren(value)) {
-            return value;
-        }
+    try {
+        const maxFields = 20;
+        let fieldCount = 0;
+        const seen = new WeakSet();
+
+        function copy(value: unknown): unknown {
+            // Handle primitives
+            if (!canHaveChildren(value)) {
+                return value;
+            }
 
-        // Check for circular references
-        if (seen.has(value as object)) {
-            return null;
-        }
-        seen.add(value as object);
+            // Check for circular references
+            if (seen.has(value as object)) {
+                return null;
+            }
+            seen.add(value as object);
 
-        // Check if we've hit the field limit
-        if (fieldCount >= maxFields) {
-            return null;
-        }
+            // Check if we've hit the field limit
+            if (fieldCount >= maxFields) {
+                return null;
+            }
+
+            if (Array.isArray(value)) {
+                const result: unknown[] = [];
+                for (const item of value) {
+                    fieldCount++;
+                    if (fieldCount >= maxFields) break;
+                    result.push(copy(item));
+                }
+                seen.delete(value);
+                return result;
+            }
 
-        if (Array.isArray(value)) {
-            const result: unknown[] = [];
-            for (const item of value) {
+            // Handle plain objects
+            const result: Record<string, unknown> = {};
+            for (const key of Object.keys(value as object)) {
                 fieldCount++;
                 if (fieldCount >= maxFields) break;
-                result.push(copy(item));
+                result[key] = copy((value as Record<string, unknown>)[key]);
             }
             seen.delete(value);
             return result;
         }
 
-        // Handle plain objects
-        const result: Record<string, unknown> = {};
-        for (const key of Object.keys(value as object)) {
-            fieldCount++;
-            if (fieldCount >= maxFields) break;
-            result[key] = copy((value as Record<string, unknown>)[key]);
-        }
-        seen.delete(value);
-        return result;
+        return copy(obj) as any;
+    } catch (e: any) {
+        return { errorCopying: e.messsage } as any;
     }
-
-    return copy(obj) as any;
 }
 function packageLogObj(args: unknown[]): LogObj {
     let logObj: LogObj = {

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -9,19 +9,24 @@ export function shimConsoleLogs() {
         let console = globalThis.console;
         let originalFnc = console[fncName];
         console[fncName] = (...args: any[]) => {
-            if (
-                args.length > 0
-                && String(args[0]).trim().length > 0
-            ) {
-                if (typeof logDisk === "function") {
-                    // Don't call it directly, so we don't get extra line debug context added to this call
-                    //  (as it wouldn't be useful, as we really want the caller)
-                    let stopDoubleShim = logDisk;
-                    stopDoubleShim(...args, { type: fncName });
+            try {
+                if (
+                    args.length > 0
+                    && String(args[0]).trim().length > 0
+                ) {
+                    if (typeof logDisk === "function") {
+                        // Don't call it directly, so we don't get extra line debug context added to this call
+                        //  (as it wouldn't be useful, as we really want the caller)
+                        let stopDoubleShim = logDisk;
+                        stopDoubleShim(...args, { type: fncName });
+                    }
                 }
+                // Filter out objects added by injectFileLocationToConsole
+                args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
+            } catch {
+                // Some arguments might throw if accessed (as they might be proxies), so
+                //  catch and ignore errors
             }
-            // Filter out objects added by injectFileLocationToConsole
-            args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
             return originalFnc(...args);
         };
     }

package/src/3-path-functions/tests/functionsTest.ts

@@ -1,135 +0,0 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { delay } from "socket-function/src/batching";
-import { magenta, red } from "socket-function/src/formatting/logColors";
-import { measureCode } from "socket-function/src/profiling/measure";
-import { getThreadKeyCert } from "../../-a-auth/certs";
-import { logErrors } from "../../errors";
-import { getPathStr1 } from "../../path";
-import { proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
-import { pathValueCommitter } from "../../0-path-value-core/PathValueController";
-import { quietCoreMode } from "../../0-path-value-core/pathValueCore";
-import { PathFunctionRunner } from "../PathFunctionRunner";
-import * as FunctionRunner from "../PathFunctionRunner";
-import * as test from "../../../../shard-app/src/data";
-import { deployFunction } from "../PathFunctionHelpers";
-
-// async function testCallFunctions() {
-//     //ActionsHistory.LOG_ACTION_HISTORY = "caller";
-
-//     quietCoreMode();
-//     // ClientWatcher.DEBUG_READS = true;
-//     // ClientWatcher.DEBUG_WRITES = true;
-//     //PathValueProxyWatcher.DEBUG = true;
-//     //PathFunctionRunner.DEBUG_CALLS = true;
-
-//     await SocketFunction.mount({ port: 0, ...await getThreadKeyCert() });
-
-//     let count = 1000 * 1000;
-//     let writesBetweenDelays = 500;
-//     let delayTime = 1000;
-//     // writesBetweenDelays = 10;
-//     //count = 1;
-//     //count = 0;
-//     // 0 + 30 * 4 = 120
-//     //count = 1000 * 100;
-//     //count = 1000 * 10;
-//     //count = 1000;
-//     //count = 100;
-//     count = 1;
-
-//     await delay(1000);
-
-//     await measureCode(async function testCallFunctions() {
-
-//         const moduleId = "880d9f63-232a-4df7-86b8-eeb711ac3e48";
-//         const functionId = "80d87119-68fc-4440-87d9-fcbab00bec22";
-//         // Deploy the function
-//         await deployFunction({
-//             domainName: "querysub.com",
-//             moduleId,
-//             functionId,
-//             filePath: "/test.ts",
-//             exportPathStr: getPathStr1("testWrite"),
-//             gitRef: "main",
-//             gitURL: "https://github.com/sliftist/shard-app.git",
-//         });
-
-
-//         let lastValue = await proxyWatcher.commitFunction({
-//             debugName: "readValue",
-//             watchFunction() {
-//                 return +test.schema()["querysub.com"].testData.test;
-//             },
-//         });
-//         let endValue = lastValue + count;
-
-//         console.log(magenta(`Start at ${lastValue}`));
-
-//         let triggerReaderCount = 0;
-//         // Watch for updates
-//         proxyWatcher.createWatcher({
-//             watchFunction: function reader() {
-//                 const testData = test.schema()["querysub.com"].testData;
-//                 let value = testData.test;
-//                 lastValue = value as number;
-//                 triggerReaderCount++;
-//             }
-//         });
-
-//         // Write function calls
-//         for (let i = 0; i < count; i++) {
-//             FunctionRunner.writeFunctionCall({
-//                 domainName: "querysub.com",
-//                 moduleId,
-//                 functionId,
-//                 args: ["test"],
-//             });
-//             if (i % (writesBetweenDelays + 1) === 0) {
-//                 // Wait a while, so FunctionRunner has a change to actually run the calls.
-//                 console.log(`Delay to give functions time to run (at ${lastValue})`);
-//                 await delay(delayTime);
-//                 console.log(`(at ${lastValue})`);
-//             }
-//         }
-
-//         await pathValueCommitter.waitForValuesToCommit();
-//         await delay(1000 * 2);
-
-//         while (!(lastValue >= endValue)) {
-//             console.log(`Waiting for ${endValue}, value is ${lastValue} (${endValue - lastValue} to go)`);
-//             await delay(1000);
-//         }
-
-//         console.log(magenta(`Finished at ${endValue}`));
-//     });
-//     console.log(`PathFunctionRunner.RUN_FINISH_COUNT = ${PathFunctionRunner.RUN_FINISH_COUNT}`);
-
-//     /*
-//     // NOTE: Temporary, to ensure we aren't leaking memory
-//     console.log(`Wait for memory to clear`);
-//     await delay(MAX_CHANGE_AGE);
-//     await delay(MAX_CHANGE_AGE);
-//     global.gc?.();
-//     console.log(`Done waiting memory to clear`);
-//     await writeHeapSnapshot(getSubFolder("heapdumps"));
-//     console.log(`Done post heap dump`);
-//     */
-
-//     process.exit();
-// }
-// if (process.argv.includes("--caller")) {
-//     logErrors(testCallFunctions());
-// }
-
-// async function watch() {
-//     await SocketFunction.mount({ port: 0, ...await getThreadKeyCert() });
-//     proxyWatcher.createWatcher({
-//         watchFunction: function reader() {
-//             const testData = test.schema()["querysub.com"].testData;
-//             console.log(red(`testData.test = ${testData.test}`));
-//         }
-//     });
-// }
-// if (process.argv.includes("--watch")) {
-//     logErrors(watch());
-// }