querysub 0.125.0 → 0.127.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.125.0",
+  "version": "0.127.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -158,6 +158,10 @@ interface WatcherOptions<Result> {
     // Triggers are also tracked if ClientWatcher.DEBUG_SOURCES
     // Sometimes we need trigger tracking for delta watchers, so this isn't just for debugging
     trackTriggers?: boolean;
+
+    // Fairly self explanatory. If there are nested createWatchers, instead of forking, we just
+    //  run them immediately.
+    inlineNestedWatchers?: boolean;
 }
 
 let harvestableReadyLoopCount = 0;
@@ -889,7 +893,9 @@ export class PathValueProxyWatcher {
         //  callers would need to handle a variable return type, which is annoying.
         //  - This means any writes are attributed to the existing call, which... is fine, if the caller wants
         //      to disconnect the call from the existing watcher, they should use `Promise.resolve().finally(...)`.
-        if (options.runImmediately && this.runningWatcher) {
+        // NOTE: We check for runImmediately as we still want to be able to create watchers in
+        //  reaction to trigger of other watchers.
+        if (this.runningWatcher && (options.runImmediately || this.runningWatcher.options.inlineNestedWatchers)) {
             // TODO: We COULD change our dispose out and change it back, if we find
             //  we often want to mutate shallow props of nested watchers.
             const outerWatcher = {
@@ -1616,12 +1622,21 @@ export class PathValueProxyWatcher {
             onWritesCommitted?: (writes: PathValue[]) => void;
         }
     ): Promise<Result> {
+        options = {
+            canWrite: true,
+            ...options,
+        };
+
+        if (this.runningWatcher?.options.inlineNestedWatchers) {
+            this.createWatcher(options);
+            return "Nested commitFunctions cannot return values" as any;
+        }
+
         let onResult!: (result: Result) => void;
         let onError!: (error: Error) => void;
         let resultPromise = new Promise<Result>((resolve, reject) => { onResult = resolve; onError = reject; });
         let anyWrites = false;
         this.createWatcher({
-            canWrite: true,
             ...options,
             onResultUpdated: (result, writes, watcher) => {
                 if (writes?.length) {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,11 +1,11 @@
 import { delay, runInfinitePoll } from "socket-function/src/batching";
 import { cache } from "socket-function/src/caching";
-import { magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { blue, magenta, red, yellow } from "socket-function/src/formatting/logColors";
 import { sha256HashBuffer, timeInHour } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getLowUint32, getShortNumber } from "../bits";
 import { registerDynamicResource } from "../diagnostics/trackResources";
-import { logErrors } from "../errors";
+import { errorToUndefined, logErrors } from "../errors";
 import { getPathDepth, getPathFromStr, getPathIndex, trimPathStrToDepth } from "../path";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
@@ -17,9 +17,11 @@ import debugbreak from "debugbreak";
 import { parseArgs } from "./PathFunctionHelpers";
 import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
-import { set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 
 export const functionSchema = rawSchema<{
     [domainName: string]: {
@@ -176,6 +178,7 @@ export class PathFunctionRunner {
         PermissionsChecker: PermissionsCheckType | undefined;
         filterSelector?: FilterSelector;
     }) {
+        SocketFunction.expose(FunctionPreloadController);
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
@@ -655,4 +658,36 @@ export class PathFunctionRunner {
         });
         PathFunctionRunner.RUN_FINISH_COUNT++;
     }
-}
+}
+
+export async function preloadFunctions(specs: FunctionSpec[]) {
+    let nodeIds = await getControllerNodeIdList(FunctionPreloadController);
+    await Promise.allSettled(nodeIds.map(async nodeId => {
+        let controller = FunctionPreloadController.nodes[nodeId.nodeId];
+        console.log(blue(`Preloading functions on ${nodeId}`));
+        await errorToUndefined(controller.preloadFunctions(specs));
+        console.log(blue(`Finished preloading functions on ${nodeId}`));
+    }));
+}
+
+class FunctionPreloaderBase {
+    async preloadFunctions(specs: FunctionSpec[]) {
+        for (let spec of specs) {
+            await getModuleFromSpec(spec);
+        }
+    }
+}
+
+const FunctionPreloadController = SocketFunction.register(
+    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
+    new FunctionPreloaderBase(),
+    () => ({
+        preloadFunctions: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);

package/src/3-path-functions/pathFunctionLoader.ts

@@ -263,7 +263,6 @@ function getCallstackFiles(): string[] {
 async function getModuleFromSpecBase(
     spec: LoadFunctionSpec
 ): Promise<NodeJS.Module> {
-    everLoadedRepos.add(spec.gitURL);
     let hotReloadPackagePath = "";
     let path = gitURLRefMappings.get(getSpecKey(spec));
     let deployPath = "";
@@ -430,7 +429,7 @@ async function executeCommand(command: string, args: string[], options?: {
         const child = child_process.spawn(resolvedCommandPath, args, {
             cwd: options?.cwd,
             stdio: ["ignore", "pipe", "pipe"],
-            shell: process.platform === "win32",
+            shell: true,
         });
 
         let stdout = "";
@@ -459,37 +458,3 @@ async function executeCommand(command: string, args: string[], options?: {
         });
     });
 }
-
-// Helps us exclude non-function loaders (PathValueServer), which really shouldn't
-//  be loading code anyways!
-let everLoadedRepos = new Set<string>();
-
-export async function preloadFunctions(specs: FunctionSpec[]) {
-    let nodeIds = await getControllerNodeIdList(functionPreloadController);
-    await Promise.all(nodeIds.map(async nodeId => {
-        let controller = functionPreloadController.nodes[nodeId.nodeId];
-        console.log(blue(`Preloading functions on ${nodeId}`));
-        await errorToUndefined(controller.preloadFunctions(specs));
-        console.log(blue(`Finished preloading functions on ${nodeId}`));
-    }));
-}
-
-class FunctionPreloaderBase {
-    async preloadFunctions(specs: FunctionSpec[]) {
-        specs = specs.filter(spec => everLoadedRepos.has(spec.gitURL));
-        for (let spec of specs) {
-            await getModuleFromSpec(spec);
-        }
-    }
-}
-
-const functionPreloadController = SocketFunction.register(
-    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
-    new FunctionPreloaderBase(),
-    () => ({
-        preloadFunctions: {},
-    }),
-    () => ({
-        hooks: [requiresNetworkTrustHook],
-    })
-);

package/src/4-deploy/deployMain.ts

@@ -1,6 +1,6 @@
 import "../inject";
 
-import { FunctionSpec } from "../3-path-functions/PathFunctionRunner";
+import { FunctionSpec, preloadFunctions } from "../3-path-functions/PathFunctionRunner";
 import path from "path";
 import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "../3-path-functions/syncSchema";
 import { getModuleRelativePath } from "../3-path-functions/syncSchema";
@@ -15,13 +15,20 @@ import yargs from "yargs";
 import { isPublic } from "../config";
 import { deployBlock } from "./deployBlock";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { preloadFunctions } from "../3-path-functions/pathFunctionLoader";
 import { magenta } from "socket-function/src/formatting/logColors";
 import { getGitRef } from "./git";
 import { getGitURL } from "./git";
 import { setLiveDeployedHash } from "./deploySchema";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import debugbreak from "debugbreak";
+import { timeInHour } from "socket-function/src/misc";
+import { preloadUI } from "./edgeNodes";
+
 let yargObj = yargs(process.argv)
     .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
+    .option("deployonlycode", { type: "boolean", desc: "Only deploy code, not ui" })
+    .option("deployonlyui", { type: "boolean", desc: "Only deploy ui, not code" })
+    .option("notifyrefreshdelay", { type: "number", default: timeInHour * 12, desc: "The time clients have to refresh their UI after a deploy." })
     .argv
     ;
 
@@ -57,12 +64,6 @@ export async function deployMain() {
     }
     let gitRef = getGitRef(gitDir);
 
-
-    //todonext
-    // Flags to set this
-    //setLiveDeployedHash(gitRef);
-
-
     let usedModules = new Map<string, string>();
 
     for (let module of Object.values(require.cache)) {
@@ -118,13 +119,34 @@ export async function deployMain() {
 
     console.log();
     console.log();
-    console.log(magenta(`Preloading ${currentFunctions.length} functions`));
-    await preloadFunctions(currentFunctions);
+    console.log(magenta(`Preloading ${currentFunctions.length} functions & UI`));
+    await Promise.all([
+        preloadFunctions(currentFunctions),
+        preloadUI(gitRef),
+    ]);
 
     console.log();
     console.log();
     console.log(magenta(`Deploying ${currentFunctions.length} functions`));
-    await replaceFunctions({ domainName, functions: currentFunctions, });
+
+    let refreshThresholdTime = Date.now() + yargObj.notifyrefreshdelay;
+
+    debugbreak(2);
+    debugger;
+    //todonext;
+    // VERIFY that the nested commit functions run correctly WITH the correct (atomic) flags
+    await proxyWatcher.commitFunction({
+        debugName: "setLiveDeployedHash",
+        inlineNestedWatchers: true,
+        watchFunction: () => {
+            if (!yargObj.deployonlycode) {
+                void setLiveDeployedHash({ hash: gitRef, refreshThresholdTime, });
+            }
+            if (!yargObj.deployonlyui) {
+                void replaceFunctions({ domainName, functions: currentFunctions, });
+            }
+        },
+    });
 
     console.log(magenta(`FINISHED DEPLOY`));
 }

package/src/4-deploy/deploySchema.ts

@@ -20,15 +20,21 @@ export const deploySchema = rawSchema<{
 export interface DeployedHash {
     hash: string;
     time: number;
+    refreshThresholdTime: number;
 }
 
 
-export function setLiveDeployedHash(hash: string) {
+export function setLiveDeployedHash(config: {
+    hash: string;
+    refreshThresholdTime: number;
+}) {
+    const { hash, refreshThresholdTime } = config;
     proxyWatcher.writeOnly({
         watchFunction: function writeCall() {
             let hashObj: DeployedHash = {
                 hash,
                 time: Date.now(),
+                refreshThresholdTime,
             };
             deploySchema()[getDomain()].deploy.live = hashObj;
             deploySchema()[getDomain()].deploy.history[nextId()] = hashObj;

package/src/4-deploy/edgeNodes.ts

@@ -16,6 +16,10 @@ import { getGitRef, getGitURL } from "./git";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import path from "path";
 import debugbreak from "debugbreak";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
+import { blue } from "socket-function/src/formatting/logColors";
+import { errorToUndefined } from "../errors";
 
 const UPDATE_POLL_INTERVAL = timeInMinute;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -51,6 +55,7 @@ export type EdgeNodesIndex = {
     edgeNodes: EdgeNodeConfig[];
 };
 
+
 // IMPORTANT! The EdgeNodeConfig MUST be immutable, otherwise every node has to read every EdgeNodeConfig
 //  every poll, which is too much reading.
 export type EdgeNodeConfig = {
@@ -100,17 +105,11 @@ export async function registerEdgeNode(config: {
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
 
-    //todonext
-    //  Testing
-    //if (!isLocal())
-    {
+    if (!isLocal()) {
         await loadEntryPointsByHash({
             host: config.host,
             entryPaths: config.entryPaths,
-            //todonext
-            //  Testing
-            //hash: gitHash,
-            hash: "8554dd139250ea3fe11f2f3551e8d59ab7122978",
+            hash: gitHash,
         });
     }
 
@@ -131,30 +130,8 @@ async function loadEntryPointsByHash(config: {
         gitRef: gitHash,
         FilePath: entryPaths[0],
         FunctionId: "entryPoint",
-        //todonext
-        //  Testing
-        noLocal: true,
     });
 
-    //todonext;
-    //  2) Run aipaint normally. This should have a later bootTime, and therefore be picked,
-    //      so it should test our browser loading other gitHashes
-    //  3) Setup the deployMain code to set the live hash
-    //      - Inside of replaceFunctions, via passing a callback function to it. The nested commits SHOULD
-    //          just work?
-    //          - IMPORTANT! Step through the code, to ensure the nested flags are set correctly. If not...
-    //              update the proxy watcher to set them properly...
-    //      - With all the flags (that only do one or the other)
-    //      - Add preloading capability, so we can preload the hashes before we deploy
-    //  3) Dynamically watch the live hash, and load new hashes (deploying their services)
-    //  - Test by pushing, deploying, and refreshing (we should prefer to latest service)
-    //  3.1) If local, DO NOT watch the live hash
-    //  5) Write the live hash to public backblaze, and have the client read it, and prefer that hash
-    //  6) Pass the live hash to the client with the edge node
-    //  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
-    //  - Test by pushing, deploying, and then we should immediately see a notify to refresh
-
-
     let depth = entryPaths[0].replaceAll("\\", "/").split("/").filter(x => x && x !== ".").length;
 
     let firstPath = path.resolve(module.filename);
@@ -177,6 +154,8 @@ async function loadEntryPointsByHash(config: {
     registeredEdgeNode = edgeNodeConfig;
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
+
+    SocketFunction.expose(EdgeNodeController);
 }
 
 export const getEdgeNodeConfigURL = lazy(async () => {
@@ -254,6 +233,46 @@ async function updateEdgeNodesFile() {
     await edgeNodeStorage.set("edge-nodes-index.json", Buffer.from(JSON.stringify(newEdgeNodeIndex)));
 }
 
+export async function preloadUI(hash: string) {
+    let nodeIds = await getControllerNodeIdList(EdgeNodeController);
+    await Promise.allSettled(nodeIds.map(async nodeId => {
+        let controller = EdgeNodeController.nodes[nodeId.nodeId];
+        console.log(blue(`Preloading functions on ${nodeId}`));
+        await errorToUndefined(controller.preloadUI({ hash }));
+        console.log(blue(`Finished preloading functions on ${nodeId}`));
+    }));
+}
+
+class EdgeNodeControllerBase {
+    public async preloadUI(config: {
+        hash: string;
+    }) {
+        const { hash } = config;
+        if (!registeredEdgeNode || !canHaveChildren(registeredEdgeNode)) {
+            throw new Error(`Somehow we have no registered edge node. How did we even expose this controller?`);
+        }
+        await loadEntryPointsByHash({
+            host: registeredEdgeNode.host,
+            entryPaths: registeredEdgeNode.entryPaths,
+            hash,
+        });
+    }
+}
+
+const EdgeNodeController = SocketFunction.register(
+    "EdgeNodeController-e0110af9-f8c4-42f3-ad11-633f8e43abdd",
+    new EdgeNodeControllerBase(),
+    () => ({
+        preloadUI: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+
 // async function main() {
 //     await loadEntryPointsByHash({
 //         host: "127-0-0-1.querysub.com:1111",
@@ -261,4 +280,26 @@ async function updateEdgeNodesFile() {
 //         hash: "7b1eb4934d5bdfe6d0f7076049c4b7decad4e645",
 //     });
 // }
-// main().catch(console.error).finally(() => process.exit());
+// main().catch(console.error).finally(() => process.exit());
+
+
+
+//todonext;
+//  3) Setup the deployMain code to set the live hash
+//      - Flags to decide what to set
+//      - Inside of replaceFunctions, via passing a callback function to it. The nested commits SHOULD
+//          just work?
+//          - IMPORTANT! Step through the code, to ensure the nested flags are set correctly. If not...
+//              update the proxy watcher to set them properly...
+//      - With all the flags (that only do one or the other)
+//      - Add preloading capability, so we can preload the hashes before we deploy
+
+//  3) Dynamically watch the live hash, and load new hashes (deploying their services)
+//  - Test by pushing, deploying, and refreshing (we should prefer to latest service)
+//  3.1) If local, DO NOT watch the live hash
+
+//  5) Write the live hash to public backblaze, and have the client read it, and prefer that hash
+//  6) Pass the live hash to the client with the edge node
+//  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
+//  - Test by pushing, deploying, and then we should immediately see a notify to refresh
+//  8) VERIFY writing and permissions stillw orks