querysub 0.437.0 → 0.439.0

package/src/3-path-functions/pathFunctionLoader.ts

@@ -1,516 +1,516 @@
-import { getSubFolder } from "../fs";
-import { FunctionSpec } from "./PathFunctionRunner";
-import child_process from "child_process";
-import fs from "fs";
-import { blue, magenta, red } from "socket-function/src/formatting/logColors";
-import debugbreak from "debugbreak";
-import { cache, lazy } from "socket-function/src/caching";
-import { batchFunction, delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
-import { errorToUndefined, logErrors } from "../errors";
-import crypto from "crypto";
-import { MaybePromise } from "socket-function/src/types";
-import yargs from "yargs";
-import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
-import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
-import { getPathStr2, getPathStr3 } from "../path";
-import { consistentHash } from "../misc/hash";
-import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
-import { devDebugbreak, isLocal, isPublic, setIsDynamicallyLoading } from "../config";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
-import { sha256 } from "js-sha256";
-import os from "os";
-import { formatTime } from "socket-function/src/formatting/format";
-import path from "path";
-
-export type LoadFunctionSpec = {
-    gitURL: string;
-    gitRef: string;
-    FilePath: string;
-    FunctionId: string;
-    // NOTE: Ignores the local flag temporarily. For testing.
-    noLocal?: boolean;
-};
-
-let watchers = new Set<SyncWatcher>();
-let ensureDisposingDeadWatchers = lazy(() => {
-    runInfinitePoll(MAX_CHANGE_AGE, function disposeDeadWatchers() {
-        for (let watcher of watchers) {
-            if (watcher.disposed) {
-                watchers.delete(watcher);
-            }
-        }
-    });
-});
-export function watchModuleHotreloads(watcher: SyncWatcher) {
-    watchers.add(watcher);
-    ensureDisposingDeadWatchers();
-}
-
-const getLocalPathRemapping = lazy((): { [gitUrl: string]: string } => {
-    let localPaths: string[] = [];
-    if (!isPublic()) {
-        localPaths.push(process.cwd().replaceAll(/\\/g, "/") + "/");
-    }
-    if (!localPaths.length) return {};
-    let mapping: { [gitUrl: string]: string } = Object.create(null);
-    for (let path of localPaths) {
-        if (!path.endsWith("/")) {
-            path += "/";
-        }
-        let packageJSON = fs.readFileSync(path + "package.json", "utf8");
-        let packageJSONObj = JSON.parse(packageJSON);
-        let repo = packageJSONObj.repository as { type: string, url: string } | undefined;
-        if (!repo) throw new Error(`No "repository" property in package.json at ${path}`);
-        if (repo.type !== "git" && repo.type !== "git+https" && repo.type !== "https") {
-            throw new Error(`Repository type ${JSON.stringify(repo.type)} is not supported yet. Only "git", "git+https", and "https" are presently supported.`);
-        }
-        mapping[repo.url] = path;
-        // https://github.com/sliftist/qs-cyoa.git
-        // git@github.com:sliftist/qs-cyoa.git
-        mapping[repo.url.replace(/^https:\/\/([^\/]+)\//, "git@$1:")] = path;
-        // Go from git to https as well, as it might be https
-        mapping[repo.url.replace(/^git@([^:]+):(.+)$/, "https://$1/$2")] = path;
-    }
-    return mapping;
-});
-
-// If we have so many different functions that we can't cache them all... then we are going
-//  to have a problem! Also... this cache is only the function, not the require.cache, which
-//  isn't being cleared anyways, so THIS isn't really where we would leak.
-export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Module> {
-    return getModuleFromConfig(spec);
-}
-let moduleCache = new Map<string, {
-    result: MaybePromise<NodeJS.Module>;
-    error: Error | undefined;
-}>();
-function getSpecKey(spec: LoadFunctionSpec) {
-    // Only include these specific keys AND order them, so the hash hits no matter how this function is called
-    spec = {
-        FilePath: spec.FilePath,
-        gitURL: spec.gitURL,
-        gitRef: spec.gitRef,
-        FunctionId: spec.FunctionId,
-        noLocal: spec.noLocal,
-    };
-    return JSON.stringify(spec);
-}
-export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS.Module> {
-    let key = getSpecKey(spec);
-    let value = moduleCache.get(key);
-    if (!value) {
-        let promise = getModuleFromSpecBase(spec);
-        value = { result: promise, error: undefined };
-        moduleCache.set(key, value);
-        promise.then(
-            result => {
-                moduleCache.set(key, { result, error: undefined });
-            },
-            error => {
-                moduleCache.set(key, { result: null as any, error });
-            }
-        );
-    }
-    if (value.error) throw value.error;
-    return value.result;
-}
-
-let gitURLRefMappings = new Map<string, string>();
-export function setGitURLMapping(config: {
-    spec: FunctionSpec;
-    resolvedPath: string;
-}) {
-    gitURLRefMappings.set(getSpecKey(config.spec), config.resolvedPath);
-}
-
-const loadTimeIndicatorFileName = "loadTimeIndicator-71cd93ba-1667-49ac-9206-b27930bbd983";
-
-/** spec => path that we can use with require */
-let moduleResolver = async (spec: {
-    gitURL: string;
-    gitRef: string;
-}) => {
-    if (!isPublic()) {
-        // Probably a bug. The local path should have matched.
-        devDebugbreak();
-    }
-    let gitURL = spec.gitURL;
-    let urlForPath = gitURL;
-
-    if (urlForPath.startsWith("http")) {
-        // Switch to a git url... incase the repo is private? Or something?
-        // TODO: We should only do this selectively, as I think we can't sync this
-        //  way without owning the repo, or... something?
-
-        let url = new URL(urlForPath);
-        gitURL = "git@" + url.host + ":" + url.pathname.slice(1);
-        urlForPath = gitURL;
-    }
-    urlForPath = urlForPath.replaceAll("@", "/");
-    urlForPath = urlForPath.replaceAll(":", "/");
-    if (urlForPath.endsWith(".git")) {
-        urlForPath = urlForPath.slice(0, -".git".length);
-    }
-    let repoPath = getSubFolder("synced_repos") + urlForPath + "/" + spec.gitRef + "/";
-
-    const lockFolder = getSubFolder("synced_repos_locks");
-    let lockPath = lockFolder + sha256(repoPath).slice(0, 16) + ".loadinglock";
-    let exists = fs.existsSync(repoPath);
-    if (exists && !fs.existsSync(repoPath + loadTimeIndicatorFileName)) {
-        exists = false;
-    }
-    if (!exists) {
-        await getFileLock(lockPath, async () => {
-            if (fs.existsSync(repoPath + loadTimeIndicatorFileName)) return;
-
-            let time = Date.now();
-
-            // Remove any previous attempt to sync it
-            if (fs.existsSync(repoPath)) {
-                await fs.promises.rm(repoPath, { recursive: true });
-            }
-            // Clone it
-            await executeCommand("git", ["clone", gitURL, repoPath]);
-            await executeCommand("git", ["reset", "--hard", spec.gitRef], { cwd: repoPath });
-
-            // Yarn install
-            await executeCommand("yarn", ["install"], { cwd: repoPath });
-
-            // Delete querysub, and replace it with a symlink. Otherwise the synchronization code
-            //  will run again, and a lot of setup code will run again, etc, and nothing will work correctly.
-            let querysubPath = repoPath + "node_modules/querysub";
-            await fs.promises.rm(querysubPath, { recursive: true });
-
-            let actualQuerysubPath = path.resolve("./node_modules/querysub");
-            await fs.promises.symlink(actualQuerysubPath, querysubPath, "junction");
-
-            // Mark it as loaded. If we don't reach this point we will move the folder and try again next time
-            await fs.promises.writeFile(repoPath + loadTimeIndicatorFileName, Date.now() + "");
-
-            time = Date.now() - time;
-            console.log(blue(`Cloned and yarn installed repo`), { gitURL, time });
-        });
-    }
-
-    return repoPath;
-};
-
-const lockHeartbeatInterval = timeInSecond * 15;
-const lockExpiryTime = timeInSecond * 60;
-const lockCheckTime = 500;
-async function tryGetFileLock(file: string) {
-    let expiryTime = Date.now() - lockExpiryTime;
-    try {
-        let lastHeartbeat = Number((await fs.promises.readFile(file, "utf8")).split("-")[0]) || 0;
-        if (lastHeartbeat > expiryTime) return false;
-    } catch { }
-    let heartbeatWrite = Date.now() + "-" + nextId();
-    let dir = path.dirname(file);
-    if (!fs.existsSync(dir)) {
-        await fs.promises.mkdir(dir, { recursive: true });
-    }
-    await fs.promises.writeFile(file, heartbeatWrite);
-    await delay(lockCheckTime);
-    let readBack = await fs.promises.readFile(file, "utf8");
-    if (readBack !== heartbeatWrite) return false;
-    return true;
-}
-async function getFileLock(file: string, callback: () => Promise<void>) {
-    console.log(magenta(`Getting file lock: ${file}`));
-    while (true) {
-        if (await tryGetFileLock(file)) {
-            console.log(magenta(`Got file lock: ${file}`));
-            let done = false;
-            logErrors((async () => {
-                while (!done) {
-                    await delay(lockHeartbeatInterval);
-                    console.log(magenta(`Not done with file lock, writing heartbeat: ${file}`));
-                    await fs.promises.writeFile(file, Date.now() + "-" + nextId());
-                }
-            })());
-            try {
-                await callback();
-            } finally {
-                await fs.promises.unlink(file);
-                done = true;
-                console.log(magenta(`Releasing file lock: ${file}`));
-            }
-            return;
-        }
-        console.log(magenta(`Waiting for lock file lock: ${file}`));
-        await delay(lockHeartbeatInterval);
-    }
-}
-
-export function isDynamicModule(module: NodeJS.Module): boolean {
-    return isDynamicModulePath(module.filename);
-}
-export function isDynamicModulePath(path: string): boolean {
-    let parts = path.replaceAll("\\", "/").split("/");
-    return parts.includes("node_modules") || parts.includes("synced_repos");
-}
-/** Annoying, and slower than isDynamicModule, but... necessary. We can't expect the user to call isDynamicModule,
- *      and our functions will all resolve to the root on purpose, so... we need to check the callstack!
- */
-export function isCallerDynamicModule(): boolean {
-    return getCallstackFiles().some(isDynamicModulePath);
-}
-function getCallstackFiles(): string[] {
-    let stack = new Error().stack;
-    if (!stack) return [];
-    let lines = stack.split("\n");
-    let files = lines.map(line => {
-        let match = line.match(/\(([^)]+)\)/);
-        if (!match) return "";
-        return match[1];
-    });
-    return files;
-}
-
-let importBlockers = new Set<Promise<unknown>>();
-export function registerImportBlockers(blocker: Promise<unknown>) {
-    importBlockers.add(blocker);
-    void blocker.finally(() => {
-        importBlockers.delete(blocker);
-    });
-}
-export async function waitForImportBlockers() {
-    while (importBlockers.size > 0) {
-        await Promise.all(Array.from(importBlockers));
-    }
-}
-
-function getDirname(fileName: string) {
-    fileName = fileName.replaceAll("\\", "/");
-    let lastSlash = fileName.lastIndexOf("/");
-    if (lastSlash === -1) return "";
-    return fileName.slice(0, lastSlash);
-}
-
-// On import every parent folder is set to the spec. For root folders this makes
-//  the spec useless, but for unique folders it will only be set once, making it very useful!
-// "overlapping" means multiple folders mapped to this spec
-let folderToSpec = new Map<string, LoadFunctionSpec | "overlapping">();
-function registerSpec(fileName: string, spec: LoadFunctionSpec | "overlapping") {
-    let folder = getDirname(fileName);
-    while (true) {
-        if (folderToSpec.has(folder)) {
-            folderToSpec.set(folder, "overlapping");
-        } else {
-            folderToSpec.set(folder, spec);
-        }
-        let parentFolder = getDirname(folder);
-        if (parentFolder === folder) {
-            break;
-        }
-        folder = parentFolder;
-    }
-}
-
-export function getSpecFromModule(module: NodeJS.Module): LoadFunctionSpec | undefined {
-    let folder = getDirname(module.filename);
-    while (true) {
-        let spec = folderToSpec.get(folder);
-        if (spec && typeof spec !== "string") return spec;
-        let parentFolder = getDirname(folder);
-        if (parentFolder === folder) break;
-        folder = parentFolder;
-    }
-    if (!module.parent) return undefined;
-    // Keep going up to our parent? This shouldn't really happen, but... maybe it will?
-    return getSpecFromModule(module.parent);
-}
-
-async function getModuleFromSpecBase(
-    spec: LoadFunctionSpec
-): Promise<NodeJS.Module> {
-    console.log(blue(`Loading module for ${JSON.stringify(spec)}`));
-    // Register ourself as overlapping, to prevent ambient modules from being set in this way.
-    registerSpec(module.filename, "overlapping");
-
-    let hotReloadPackagePath = "";
-    let path = gitURLRefMappings.get(getSpecKey(spec));
-    let deployPath = "";
-    if (!path) {
-        // Sync the git repo, `yarn install --ignore-scripts`, require the path, get the export, and then return that function.
-        let packagePath = !spec.noLocal && getLocalPathRemapping()[spec.gitURL] || "";
-        hotReloadPackagePath = packagePath;
-        if (!packagePath) {
-            packagePath = await moduleResolver(spec);
-        }
-
-        if (!packagePath.endsWith("/")) {
-            packagePath += "/";
-        }
-
-        let specFilePath = spec.FilePath;
-        if (specFilePath.startsWith("/")) {
-            specFilePath = specFilePath.slice(1);
-        }
-        path = packagePath + specFilePath;
-        deployPath = packagePath + "deploy.ts";
-    }
-
-    registerSpec(path, spec);
-
-    console.log(blue(`require(${JSON.stringify(path)})`));
-    try {
-        await SocketFunction.ignoreExposeCalls(async () => {
-            await setIsDynamicallyLoading(async () => {
-                // Import deploy, which should always exist, and provides a consistent
-                //  import order, fixing a lot of cyclic / module level code logic issues.
-                if (deployPath) {
-                    await (require as any)(deployPath, true);
-                }
-                // NOTE: The true tells require to not warn about the async loading
-                await (require as any)(path, true);
-
-                await waitForImportBlockers();
-            });
-        });
-    } catch (e: any) {
-        throw new Error(`Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
-    }
-
-
-    let moduleId = require.resolve(path) || path;
-    let moduleImported = require.cache[moduleId];
-    if (!moduleImported) {
-        debugbreak(2);
-        debugger;
-        throw new Error(`Module not found: ${moduleId} (for ${spec.FunctionId})`);
-    }
-
-    if (!isPublic()) {
-        if (hotReloadPackagePath) {
-            hotReloadUnderPath(hotReloadPackagePath);
-        }
-    }
-
-    return moduleImported;
-}
-
-
-
-
-// Hot reload at or under the path
-const hotReloadUnderPath = cache((path: string) => {
-    console.log(magenta(`Hot reloading under path: ${path}`));
-    path = path.replace(/\\/g, "/");
-    for (let module of Object.values(require.cache)) {
-        if (!module) continue;
-        let modulePath = module.filename.replace(/\\/g, "/");
-        if (modulePath.startsWith(path)) {
-            hotReloadModule(module);
-        }
-    }
-});
-const hotReloadModule = cache((module: NodeJS.Module) => {
-    fs.watch(module.filename, () => {
-        logErrors(hotreloadIfChanged(module));
-    });
-});
-const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged" }, async (modules: NodeJS.Module[]) => {
-    let changedModules = new Set<NodeJS.Module>();
-    for (let module of modules) {
-        let newContents = await fs.promises.readFile(module.filename, "utf8");
-        let newSHA256 = crypto.createHash("sha256").update(newContents).digest("hex");
-        if (newSHA256 !== module.sourceSHA256) {
-            changedModules.add(module);
-        }
-    }
-    console.log(magenta(`Hot reloading changed modules:`));
-    for (let module of changedModules) {
-        if (module.updateContents) {
-            let justContents = module.hotreload === false || isNode() && module.noserverhotreload;
-            console.log(magenta(`        ${module.filename} ${justContents && "(no re-evaluate)" || ""}`));
-            setExternalHotReloading(true);
-            try {
-                try {
-                    module.updateContents();
-                } catch (e) {
-                    console.error(red(`Error when loading module contents ${module.filename}`));
-                    console.error(e);
-                }
-                if (!justContents) {
-                    module.loaded = false;
-                    try {
-                        module.load(module.id);
-                    } catch (e) {
-                        module.load(module.id);
-                        console.error(red(`Error when hot reloading ${module.filename}`));
-                        console.error(e);
-                    }
-                }
-            } finally {
-                setExternalHotReloading(false);
-            }
-        } else {
-            console.log(red(`        (skipping due to missing updateContents functions) ${module.filename}`));
-        }
-    }
-
-    for (let watcher of watchers) {
-        if (watcher.disposed) {
-            watcher.dispose();
-        } else {
-            watcher.explicitlyTrigger();
-        }
-    }
-});
-
-async function which(command: string): Promise<string> {
-    let whichOrWhere = os.platform() === "win32" ? "where" : "which";
-    let path = child_process.execSync(`${whichOrWhere} ${command}`).toString().trim().replaceAll("\r", "").split("\n")[0].trim().replaceAll("\\", "/");
-    if (!path) {
-        throw new Error(`Command ${command} not found`);
-    }
-    path = `"${path}"`;
-    return path;
-}
-
-async function executeCommand(command: string, args: string[], options?: {
-    cwd?: string;
-}): Promise<string> {
-    let resolvedCommandPath = await which(command);
-    let debug = `${resolvedCommandPath} ${args.join(" ")}`;
-    if (options?.cwd) {
-        debug += ` (in ${options.cwd})`;
-    }
-    console.log(`Running command: ${debug}`);
-    return new Promise((resolve, reject) => {
-        const child = child_process.spawn(resolvedCommandPath, args, {
-            cwd: options?.cwd,
-            stdio: ["ignore", "pipe", "pipe"],
-            shell: true,
-        });
-
-        let stdout = "";
-        let stderr = "";
-
-        child.stdout.on("data", (data) => {
-            console.log(data.toString());
-            stdout += data.toString();
-        });
-
-        child.stderr.on("data", (data) => {
-            console.log(data.toString());
-            stderr += data.toString();
-        });
-
-        child.on("error", (error) => {
-            reject(new Error(`Failed to execute command ${debug}: ${error.message}`));
-        });
-
-        child.on("close", (code) => {
-            if (code !== 0) {
-                reject(new Error(`Command ${debug} failed with code ${code}.\nstdout: ${JSON.stringify(stdout)}\nstderr: ${JSON.stringify(stderr)}`));
-                return;
-            }
-            resolve(stdout.trim());
-        });
-    });
-}
+import { getSubFolder } from "../fs";
+import { FunctionSpec } from "./PathFunctionRunner";
+import child_process from "child_process";
+import fs from "fs";
+import { blue, magenta, red } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { cache, lazy } from "socket-function/src/caching";
+import { batchFunction, delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { errorToUndefined, logErrors } from "../errors";
+import crypto from "crypto";
+import { MaybePromise } from "socket-function/src/types";
+import yargs from "yargs";
+import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
+import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
+import { getPathStr2, getPathStr3 } from "../path";
+import { consistentHash } from "../misc/hash";
+import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
+import { devDebugbreak, isLocal, isPublic, setIsDynamicallyLoading } from "../config";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
+import { sha256 } from "js-sha256";
+import os from "os";
+import { formatTime } from "socket-function/src/formatting/format";
+import path from "path";
+
+export type LoadFunctionSpec = {
+    gitURL: string;
+    gitRef: string;
+    FilePath: string;
+    FunctionId: string;
+    // NOTE: Ignores the local flag temporarily. For testing.
+    noLocal?: boolean;
+};
+
+let watchers = new Set<SyncWatcher>();
+let ensureDisposingDeadWatchers = lazy(() => {
+    runInfinitePoll(MAX_CHANGE_AGE, function disposeDeadWatchers() {
+        for (let watcher of watchers) {
+            if (watcher.disposed) {
+                watchers.delete(watcher);
+            }
+        }
+    });
+});
+export function watchModuleHotreloads(watcher: SyncWatcher) {
+    watchers.add(watcher);
+    ensureDisposingDeadWatchers();
+}
+
+const getLocalPathRemapping = lazy((): { [gitUrl: string]: string } => {
+    let localPaths: string[] = [];
+    if (!isPublic()) {
+        localPaths.push(process.cwd().replaceAll(/\\/g, "/") + "/");
+    }
+    if (!localPaths.length) return {};
+    let mapping: { [gitUrl: string]: string } = Object.create(null);
+    for (let path of localPaths) {
+        if (!path.endsWith("/")) {
+            path += "/";
+        }
+        let packageJSON = fs.readFileSync(path + "package.json", "utf8");
+        let packageJSONObj = JSON.parse(packageJSON);
+        let repo = packageJSONObj.repository as { type: string, url: string } | undefined;
+        if (!repo) throw new Error(`No "repository" property in package.json at ${path}`);
+        if (repo.type !== "git" && repo.type !== "git+https" && repo.type !== "https") {
+            throw new Error(`Repository type ${JSON.stringify(repo.type)} is not supported yet. Only "git", "git+https", and "https" are presently supported.`);
+        }
+        mapping[repo.url] = path;
+        // https://github.com/sliftist/qs-cyoa.git
+        // git@github.com:sliftist/qs-cyoa.git
+        mapping[repo.url.replace(/^https:\/\/([^\/]+)\//, "git@$1:")] = path;
+        // Go from git to https as well, as it might be https
+        mapping[repo.url.replace(/^git@([^:]+):(.+)$/, "https://$1/$2")] = path;
+    }
+    return mapping;
+});
+
+// If we have so many different functions that we can't cache them all... then we are going
+//  to have a problem! Also... this cache is only the function, not the require.cache, which
+//  isn't being cleared anyways, so THIS isn't really where we would leak.
+export function getModuleFromSpec(spec: FunctionSpec): MaybePromise<NodeJS.Module> {
+    return getModuleFromConfig(spec);
+}
+let moduleCache = new Map<string, {
+    result: MaybePromise<NodeJS.Module>;
+    error: Error | undefined;
+}>();
+function getSpecKey(spec: LoadFunctionSpec) {
+    // Only include these specific keys AND order them, so the hash hits no matter how this function is called
+    spec = {
+        FilePath: spec.FilePath,
+        gitURL: spec.gitURL,
+        gitRef: spec.gitRef,
+        FunctionId: spec.FunctionId,
+        noLocal: spec.noLocal,
+    };
+    return JSON.stringify(spec);
+}
+export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS.Module> {
+    let key = getSpecKey(spec);
+    let value = moduleCache.get(key);
+    if (!value) {
+        let promise = getModuleFromSpecBase(spec);
+        value = { result: promise, error: undefined };
+        moduleCache.set(key, value);
+        promise.then(
+            result => {
+                moduleCache.set(key, { result, error: undefined });
+            },
+            error => {
+                moduleCache.set(key, { result: null as any, error });
+            }
+        );
+    }
+    if (value.error) throw value.error;
+    return value.result;
+}
+
+let gitURLRefMappings = new Map<string, string>();
+export function setGitURLMapping(config: {
+    spec: FunctionSpec;
+    resolvedPath: string;
+}) {
+    gitURLRefMappings.set(getSpecKey(config.spec), config.resolvedPath);
+}
+
+const loadTimeIndicatorFileName = "loadTimeIndicator-71cd93ba-1667-49ac-9206-b27930bbd983";
+
+/** spec => path that we can use with require */
+let moduleResolver = async (spec: {
+    gitURL: string;
+    gitRef: string;
+}) => {
+    if (!isPublic()) {
+        // Probably a bug. The local path should have matched.
+        devDebugbreak();
+    }
+    let gitURL = spec.gitURL;
+    let urlForPath = gitURL;
+
+    if (urlForPath.startsWith("http")) {
+        // Switch to a git url... incase the repo is private? Or something?
+        // TODO: We should only do this selectively, as I think we can't sync this
+        //  way without owning the repo, or... something?
+
+        let url = new URL(urlForPath);
+        gitURL = "git@" + url.host + ":" + url.pathname.slice(1);
+        urlForPath = gitURL;
+    }
+    urlForPath = urlForPath.replaceAll("@", "/");
+    urlForPath = urlForPath.replaceAll(":", "/");
+    if (urlForPath.endsWith(".git")) {
+        urlForPath = urlForPath.slice(0, -".git".length);
+    }
+    let repoPath = getSubFolder("synced_repos") + urlForPath + "/" + spec.gitRef + "/";
+
+    const lockFolder = getSubFolder("synced_repos_locks");
+    let lockPath = lockFolder + sha256(repoPath).slice(0, 16) + ".loadinglock";
+    let exists = fs.existsSync(repoPath);
+    if (exists && !fs.existsSync(repoPath + loadTimeIndicatorFileName)) {
+        exists = false;
+    }
+    if (!exists) {
+        await getFileLock(lockPath, async () => {
+            if (fs.existsSync(repoPath + loadTimeIndicatorFileName)) return;
+
+            let time = Date.now();
+
+            // Remove any previous attempt to sync it
+            if (fs.existsSync(repoPath)) {
+                await fs.promises.rm(repoPath, { recursive: true });
+            }
+            // Clone it
+            await executeCommand("git", ["clone", gitURL, repoPath]);
+            await executeCommand("git", ["reset", "--hard", spec.gitRef], { cwd: repoPath });
+
+            // Yarn install
+            await executeCommand("yarn", ["install"], { cwd: repoPath });
+
+            // Delete querysub, and replace it with a symlink. Otherwise the synchronization code
+            //  will run again, and a lot of setup code will run again, etc, and nothing will work correctly.
+            let querysubPath = repoPath + "node_modules/querysub";
+            await fs.promises.rm(querysubPath, { recursive: true });
+
+            let actualQuerysubPath = path.resolve("./node_modules/querysub");
+            await fs.promises.symlink(actualQuerysubPath, querysubPath, "junction");
+
+            // Mark it as loaded. If we don't reach this point we will move the folder and try again next time
+            await fs.promises.writeFile(repoPath + loadTimeIndicatorFileName, Date.now() + "");
+
+            time = Date.now() - time;
+            console.log(blue(`Cloned and yarn installed repo`), { gitURL, time });
+        });
+    }
+
+    return repoPath;
+};
+
+const lockHeartbeatInterval = timeInSecond * 15;
+const lockExpiryTime = timeInSecond * 60;
+const lockCheckTime = 500;
+async function tryGetFileLock(file: string) {
+    let expiryTime = Date.now() - lockExpiryTime;
+    try {
+        let lastHeartbeat = Number((await fs.promises.readFile(file, "utf8")).split("-")[0]) || 0;
+        if (lastHeartbeat > expiryTime) return false;
+    } catch { }
+    let heartbeatWrite = Date.now() + "-" + nextId();
+    let dir = path.dirname(file);
+    if (!fs.existsSync(dir)) {
+        await fs.promises.mkdir(dir, { recursive: true });
+    }
+    await fs.promises.writeFile(file, heartbeatWrite);
+    await delay(lockCheckTime);
+    let readBack = await fs.promises.readFile(file, "utf8");
+    if (readBack !== heartbeatWrite) return false;
+    return true;
+}
+async function getFileLock(file: string, callback: () => Promise<void>) {
+    console.log(magenta(`Getting file lock: ${file}`));
+    while (true) {
+        if (await tryGetFileLock(file)) {
+            console.log(magenta(`Got file lock: ${file}`));
+            let done = false;
+            logErrors((async () => {
+                while (!done) {
+                    await delay(lockHeartbeatInterval);
+                    console.log(magenta(`Not done with file lock, writing heartbeat: ${file}`));
+                    await fs.promises.writeFile(file, Date.now() + "-" + nextId());
+                }
+            })());
+            try {
+                await callback();
+            } finally {
+                await fs.promises.unlink(file);
+                done = true;
+                console.log(magenta(`Releasing file lock: ${file}`));
+            }
+            return;
+        }
+        console.log(magenta(`Waiting for lock file lock: ${file}`));
+        await delay(lockHeartbeatInterval);
+    }
+}
+
+export function isDynamicModule(module: NodeJS.Module): boolean {
+    return isDynamicModulePath(module.filename);
+}
+export function isDynamicModulePath(path: string): boolean {
+    let parts = path.replaceAll("\\", "/").split("/");
+    return parts.includes("node_modules") || parts.includes("synced_repos");
+}
+/** Annoying, and slower than isDynamicModule, but... necessary. We can't expect the user to call isDynamicModule,
+ *      and our functions will all resolve to the root on purpose, so... we need to check the callstack!
+ */
+export function isCallerDynamicModule(): boolean {
+    return getCallstackFiles().some(isDynamicModulePath);
+}
+function getCallstackFiles(): string[] {
+    let stack = new Error().stack;
+    if (!stack) return [];
+    let lines = stack.split("\n");
+    let files = lines.map(line => {
+        let match = line.match(/\(([^)]+)\)/);
+        if (!match) return "";
+        return match[1];
+    });
+    return files;
+}
+
+let importBlockers = new Set<Promise<unknown>>();
+export function registerImportBlockers(blocker: Promise<unknown>) {
+    importBlockers.add(blocker);
+    void blocker.finally(() => {
+        importBlockers.delete(blocker);
+    });
+}
+export async function waitForImportBlockers() {
+    while (importBlockers.size > 0) {
+        await Promise.all(Array.from(importBlockers));
+    }
+}
+
+function getDirname(fileName: string) {
+    fileName = fileName.replaceAll("\\", "/");
+    let lastSlash = fileName.lastIndexOf("/");
+    if (lastSlash === -1) return "";
+    return fileName.slice(0, lastSlash);
+}
+
+// On import every parent folder is set to the spec. For root folders this makes
+//  the spec useless, but for unique folders it will only be set once, making it very useful!
+// "overlapping" means multiple folders mapped to this spec
+let folderToSpec = new Map<string, LoadFunctionSpec | "overlapping">();
+function registerSpec(fileName: string, spec: LoadFunctionSpec | "overlapping") {
+    let folder = getDirname(fileName);
+    while (true) {
+        if (folderToSpec.has(folder)) {
+            folderToSpec.set(folder, "overlapping");
+        } else {
+            folderToSpec.set(folder, spec);
+        }
+        let parentFolder = getDirname(folder);
+        if (parentFolder === folder) {
+            break;
+        }
+        folder = parentFolder;
+    }
+}
+
+export function getSpecFromModule(module: NodeJS.Module): LoadFunctionSpec | undefined {
+    let folder = getDirname(module.filename);
+    while (true) {
+        let spec = folderToSpec.get(folder);
+        if (spec && typeof spec !== "string") return spec;
+        let parentFolder = getDirname(folder);
+        if (parentFolder === folder) break;
+        folder = parentFolder;
+    }
+    if (!module.parent) return undefined;
+    // Keep going up to our parent? This shouldn't really happen, but... maybe it will?
+    return getSpecFromModule(module.parent);
+}
+
+async function getModuleFromSpecBase(
+    spec: LoadFunctionSpec
+): Promise<NodeJS.Module> {
+    console.log(blue(`Loading module for ${JSON.stringify(spec)}`));
+    // Register ourself as overlapping, to prevent ambient modules from being set in this way.
+    registerSpec(module.filename, "overlapping");
+
+    let hotReloadPackagePath = "";
+    let path = gitURLRefMappings.get(getSpecKey(spec));
+    let deployPath = "";
+    if (!path) {
+        // Sync the git repo, `yarn install --ignore-scripts`, require the path, get the export, and then return that function.
+        let packagePath = !spec.noLocal && getLocalPathRemapping()[spec.gitURL] || "";
+        hotReloadPackagePath = packagePath;
+        if (!packagePath) {
+            packagePath = await moduleResolver(spec);
+        }
+
+        if (!packagePath.endsWith("/")) {
+            packagePath += "/";
+        }
+
+        let specFilePath = spec.FilePath;
+        if (specFilePath.startsWith("/")) {
+            specFilePath = specFilePath.slice(1);
+        }
+        path = packagePath + specFilePath;
+        deployPath = packagePath + "deploy.ts";
+    }
+
+    registerSpec(path, spec);
+
+    console.log(blue(`require(${JSON.stringify(path)})`));
+    try {
+        await SocketFunction.ignoreExposeCalls(async () => {
+            await setIsDynamicallyLoading(async () => {
+                // Import deploy, which should always exist, and provides a consistent
+                //  import order, fixing a lot of cyclic / module level code logic issues.
+                if (deployPath) {
+                    await (require as any)(deployPath, true);
+                }
+                // NOTE: The true tells require to not warn about the async loading
+                await (require as any)(path, true);
+
+                await waitForImportBlockers();
+            });
+        });
+    } catch (e: any) {
+        throw new Error(`Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
+    }
+
+
+    let moduleId = require.resolve(path) || path;
+    let moduleImported = require.cache[moduleId];
+    if (!moduleImported) {
+        debugbreak(2);
+        debugger;
+        throw new Error(`Module not found: ${moduleId} (for ${spec.FunctionId})`);
+    }
+
+    if (!isPublic()) {
+        if (hotReloadPackagePath) {
+            hotReloadUnderPath(hotReloadPackagePath);
+        }
+    }
+
+    return moduleImported;
+}
+
+
+
+
+// Hot reload at or under the path
+const hotReloadUnderPath = cache((path: string) => {
+    console.log(magenta(`Hot reloading under path: ${path}`));
+    path = path.replace(/\\/g, "/");
+    for (let module of Object.values(require.cache)) {
+        if (!module) continue;
+        let modulePath = module.filename.replace(/\\/g, "/");
+        if (modulePath.startsWith(path)) {
+            hotReloadModule(module);
+        }
+    }
+});
+const hotReloadModule = cache((module: NodeJS.Module) => {
+    fs.watch(module.filename, () => {
+        logErrors(hotreloadIfChanged(module));
+    });
+});
+const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged" }, async (modules: NodeJS.Module[]) => {
+    let changedModules = new Set<NodeJS.Module>();
+    for (let module of modules) {
+        let newContents = await fs.promises.readFile(module.filename, "utf8");
+        let newSHA256 = crypto.createHash("sha256").update(newContents).digest("hex");
+        if (newSHA256 !== module.sourceSHA256) {
+            changedModules.add(module);
+        }
+    }
+    console.log(magenta(`Hot reloading changed modules:`));
+    for (let module of changedModules) {
+        if (module.updateContents) {
+            let justContents = module.hotreload === false || isNode() && module.noserverhotreload;
+            console.log(magenta(`        ${module.filename} ${justContents && "(no re-evaluate)" || ""}`));
+            setExternalHotReloading(true);
+            try {
+                try {
+                    module.updateContents();
+                } catch (e) {
+                    console.error(red(`Error when loading module contents ${module.filename}`));
+                    console.error(e);
+                }
+                if (!justContents) {
+                    module.loaded = false;
+                    try {
+                        module.load(module.id);
+                    } catch (e) {
+                        module.load(module.id);
+                        console.error(red(`Error when hot reloading ${module.filename}`));
+                        console.error(e);
+                    }
+                }
+            } finally {
+                setExternalHotReloading(false);
+            }
+        } else {
+            console.log(red(`        (skipping due to missing updateContents functions) ${module.filename}`));
+        }
+    }
+
+    for (let watcher of watchers) {
+        if (watcher.disposed) {
+            watcher.dispose();
+        } else {
+            watcher.explicitlyTrigger();
+        }
+    }
+});
+
+async function which(command: string): Promise<string> {
+    let whichOrWhere = os.platform() === "win32" ? "where" : "which";
+    let path = child_process.execSync(`${whichOrWhere} ${command}`).toString().trim().replaceAll("\r", "").split("\n")[0].trim().replaceAll("\\", "/");
+    if (!path) {
+        throw new Error(`Command ${command} not found`);
+    }
+    path = `"${path}"`;
+    return path;
+}
+
+async function executeCommand(command: string, args: string[], options?: {
+    cwd?: string;
+}): Promise<string> {
+    let resolvedCommandPath = await which(command);
+    let debug = `${resolvedCommandPath} ${args.join(" ")}`;
+    if (options?.cwd) {
+        debug += ` (in ${options.cwd})`;
+    }
+    console.log(`Running command: ${debug}`);
+    return new Promise((resolve, reject) => {
+        const child = child_process.spawn(resolvedCommandPath, args, {
+            cwd: options?.cwd,
+            stdio: ["ignore", "pipe", "pipe"],
+            shell: true,
+        });
+
+        let stdout = "";
+        let stderr = "";
+
+        child.stdout.on("data", (data) => {
+            console.log(data.toString());
+            stdout += data.toString();
+        });
+
+        child.stderr.on("data", (data) => {
+            console.log(data.toString());
+            stderr += data.toString();
+        });
+
+        child.on("error", (error) => {
+            reject(new Error(`Failed to execute command ${debug}: ${error.message}`));
+        });
+
+        child.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(`Command ${debug} failed with code ${code}.\nstdout: ${JSON.stringify(stdout)}\nstderr: ${JSON.stringify(stderr)}`));
+                return;
+            }
+            resolve(stdout.trim());
+        });
+    });
+}