querysub 0.9.0 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.9.0",
+    "version": "0.10.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/3-path-functions/PathFunctionHelpers.ts

@@ -9,7 +9,6 @@ import debugbreak from "debugbreak";
 import cborx from "cbor-x";
 import { lazy } from "socket-function/src/caching";
 import { secureRandom } from "../misc/random";
-import { onCallPredict } from "../4-querysub/QuerysubController";
 import { isDefined } from "../misc";
 import { blue, green, red } from "socket-function/src/formatting/logColors";
 import { getPathStr2 } from "../path";
@@ -50,6 +49,7 @@ export async function replaceFunctions(config: {
             let currentHashes = new Set(currentFunctions.map(x => JSON.stringify(x)));
 
             let prevFunctionNames = new Set(previousFunctions.map(getFunctionName));
+            let currentFunctionNames = new Set(currentFunctions.map(getFunctionName));
 
             let messages: {
                 text: string;
@@ -57,10 +57,10 @@ export async function replaceFunctions(config: {
             }[] = [];
 
             for (let previous of previousFunctions) {
-                let hash = JSON.stringify(previous);
-                if (currentHashes.has(hash)) continue;
-                messages.push({ text: `Removing  ${red(debugFunction(previous))}`, order: 3 });
-                base[previous.ModuleId].Sources[previous.FunctionId] = undefined;
+                if (!currentFunctionNames.has(getFunctionName(previous))) {
+                    messages.push({ text: `Removing  ${red(debugFunction(previous))}`, order: 3 });
+                    base[previous.ModuleId].Sources[previous.FunctionId] = undefined;
+                }
             }
 
             console.log();
@@ -78,9 +78,11 @@ export async function replaceFunctions(config: {
                 base[func.ModuleId].Sources[func.FunctionId] = atomicObjectWrite(func);
             }
 
-            sort(messages, x => -x.order);
-            for (let message of messages) {
-                console.log(message.text);
+            if (proxyWatcher.isAllSynced()) {
+                sort(messages, x => -x.order);
+                for (let message of messages) {
+                    console.log(message.text);
+                }
             }
         }
     });

package/src/3-path-functions/deployMain.ts

@@ -16,6 +16,8 @@ import yargs from "yargs";
 import { isPublic } from "../config";
 import { deployBlock } from "./deployBlock";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
+import { preloadFunctions } from "./pathFunctionLoader";
+import { magenta } from "socket-function/src/formatting/logColors";
 let yargObj = yargs(process.argv)
     .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
     .option("nogit", { type: "boolean", default: false, desc: `Disable git operations. Useful for debugging, when using --local` })
@@ -107,7 +109,17 @@ export async function deployMain() {
         }
     }
 
+    console.log();
+    console.log();
+    console.log(magenta(`Preloading ${currentFunctions.length} functions`));
+    await preloadFunctions(currentFunctions);
+
+    console.log();
+    console.log();
+    console.log(magenta(`Deploying ${currentFunctions.length} functions`));
     await replaceFunctions({ domainName, functions: currentFunctions, });
+
+    console.log(magenta(`FINISHED DEPLOY`));
 }
 function getGitURL(gitDir: string) {
     if (yargObj.nogit) return "git@github.com:nogit/nogit.git";

package/src/3-path-functions/pathFunctionLoader.ts

@@ -5,19 +5,22 @@ import fs from "fs";
 import { blue, magenta, red } from "socket-function/src/formatting/logColors";
 import debugbreak from "debugbreak";
 import { cache, lazy } from "socket-function/src/caching";
-import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
-import { logErrors } from "../errors";
+import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
+import { errorToUndefined, logErrors } from "../errors";
 import crypto from "crypto";
 import { MaybePromise } from "socket-function/src/types";
 import yargs from "yargs";
 import { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
-import { isNode, isNodeTrue } from "socket-function/src/misc";
+import { isNode, isNodeTrue, nextId, timeInSecond } from "socket-function/src/misc";
 import { getPathStr2, getPathStr3 } from "../path";
 import { consistentHash } from "../misc/hash";
 import { setExternalHotReloading } from "socket-function/hot/HotReloadController";
 import { isPublic } from "../config";
 import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getControllerNodeId, getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
+import { sha256 } from "js-sha256";
 
 // Get localPathRemappings using yargs, so it is easy to configure in multiple entry points
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -138,13 +141,31 @@ let moduleResolver = async (spec: FunctionSpec) => {
         urlForPath = urlForPath.slice(0, -".git".length);
     }
     let repoPath = getSubFolder("synced_repos") + urlForPath + "/" + spec.gitRef + "/";
-    if (!fs.existsSync(repoPath)) {
-        await executeCommand("git", ["clone", gitURL, repoPath]);
-        await executeCommand("git", ["reset", "--hard", spec.gitRef], { cwd: repoPath });
+
+    const lockFolder = getSubFolder("synced_repos_locks");
+    let lockPath = lockFolder + sha256(repoPath).slice(0, 16) + ".loadinglock";
+    let exists = fs.existsSync(repoPath);
+    if (exists) {
+        if (!await isRepoCloned(repoPath)) {
+            exists = false;
+        }
     }
-    if (!fs.existsSync(repoPath + "node_modules")) {
-        await executeCommand("yarn", ["install"], { cwd: repoPath });
+    if (!exists) {
+        await getFileLock(lockPath, async () => {
+            if (fs.existsSync(repoPath) && !await isRepoCloned(repoPath)) {
+                // Just move it, and try again
+                await fs.promises.rename(repoPath, repoPath + "_" + Date.now());
+            }
+            if (!fs.existsSync(repoPath)) {
+                await executeCommand("git", ["clone", gitURL, repoPath]);
+                await executeCommand("git", ["reset", "--hard", spec.gitRef], { cwd: repoPath });
+            }
+            if (!fs.existsSync(repoPath + "node_modules")) {
+                await executeCommand("yarn", ["install"], { cwd: repoPath });
+            }
+        });
     }
+
     let querysubPath = repoPath + "node_modules/querysub";
     if (fs.existsSync(querysubPath)) {
         // By moving querysub, it forces the repo to use the parent querysub. This is nice, as it even
@@ -157,6 +178,57 @@ let moduleResolver = async (spec: FunctionSpec) => {
     return repoPath;
 };
 
+const lockHeartbeatInterval = timeInSecond * 15;
+const lockExpiryTime = timeInSecond * 60;
+const lockCheckTime = 500;
+async function tryGetFileLock(file: string) {
+    let expiryTime = Date.now() - lockExpiryTime;
+    try {
+        let lastHeartbeat = Number((await fs.promises.readFile(file, "utf8")).split("-")[0]) || 0;
+        if (lastHeartbeat > expiryTime) return false;
+    } catch { }
+    let heartbeatWrite = Date.now() + "-" + nextId();
+    await fs.promises.writeFile(file, heartbeatWrite);
+    await delay(lockCheckTime);
+    let readBack = await fs.promises.readFile(file, "utf8");
+    if (readBack !== heartbeatWrite) return false;
+    return true;
+}
+async function getFileLock(file: string, callback: () => Promise<void>) {
+    console.log(magenta(`Getting file lock: ${file}`));
+    while (true) {
+        if (await tryGetFileLock(file)) {
+            let done = false;
+            logErrors((async () => {
+                while (!done) {
+                    await delay(lockHeartbeatInterval);
+                    console.log(magenta(`Not done with file lock, writing heartbeat: ${file}`));
+                    await fs.promises.writeFile(file, Date.now() + "-" + nextId());
+                }
+            })());
+            try {
+                await callback();
+            } finally {
+                await fs.promises.unlink(file);
+                done = true;
+                console.log(magenta(`Releasing file lock: ${file}`));
+            }
+            return;
+        }
+        console.log(magenta(`Waiting for lock file lock: ${file}`));
+        await delay(lockHeartbeatInterval);
+    }
+}
+
+async function isRepoCloned(path: string): Promise<boolean> {
+    try {
+        await executeCommand("git", ["-C", path, "rev-parse", "--is-inside-work-tree"]);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
 export function isDynamicModule(module: NodeJS.Module): boolean {
     return isDynamicModulePath(module.filename);
 }
@@ -185,6 +257,7 @@ function getCallstackFiles(): string[] {
 async function getModuleFromSpecBase(
     spec: FunctionSpec
 ): Promise<NodeJS.Module> {
+    everLoadedRepos.add(spec.gitURL);
     let hotReloadPackagePath = "";
     let path = gitURLRefMappings.get(getSpecKey(spec));
     let deployPath = "";
@@ -333,4 +406,39 @@ async function executeCommand(command: string, args: string[], options?: {
         stdio: "inherit",
         cwd: options?.cwd
     });
-}
+}
+
+
+// Helps us exclude non-function loaders (PathValueServer), which really shouldn't
+//  be loading code anyways!
+let everLoadedRepos = new Set<string>();
+
+export async function preloadFunctions(specs: FunctionSpec[]) {
+    let nodeIds = await getControllerNodeIdList(functionPreloadController);
+    await Promise.all(nodeIds.map(async nodeId => {
+        let controller = functionPreloadController.nodes[nodeId];
+        console.log(blue(`Preloading functions on ${nodeId}`));
+        await errorToUndefined(controller.preloadFunctions(specs));
+        console.log(blue(`Finished preloading functions on ${nodeId}`));
+    }));
+}
+
+class FunctionPreloaderBase {
+    async preloadFunctions(specs: FunctionSpec[]) {
+        specs = specs.filter(spec => everLoadedRepos.has(spec.gitURL));
+        for (let spec of specs) {
+            await getModuleFromSpec(spec);
+        }
+    }
+}
+
+const functionPreloadController = SocketFunction.register(
+    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
+    new FunctionPreloaderBase(),
+    () => ({
+        preloadFunctions: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);