querysub 0.126.0 → 0.127.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.126.0",
+  "version": "0.127.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,11 +1,11 @@
 import { delay, runInfinitePoll } from "socket-function/src/batching";
 import { cache } from "socket-function/src/caching";
-import { magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { blue, magenta, red, yellow } from "socket-function/src/formatting/logColors";
 import { sha256HashBuffer, timeInHour } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getLowUint32, getShortNumber } from "../bits";
 import { registerDynamicResource } from "../diagnostics/trackResources";
-import { logErrors } from "../errors";
+import { errorToUndefined, logErrors } from "../errors";
 import { getPathDepth, getPathFromStr, getPathIndex, trimPathStrToDepth } from "../path";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
@@ -17,9 +17,11 @@ import debugbreak from "debugbreak";
 import { parseArgs } from "./PathFunctionHelpers";
 import { PERMISSIONS_FUNCTION_ID, getExportPath } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
-import { set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { FilterSelector, Filterable, doesMatch } from "../misc/filterable";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 
 export const functionSchema = rawSchema<{
     [domainName: string]: {
@@ -176,6 +178,7 @@ export class PathFunctionRunner {
         PermissionsChecker: PermissionsCheckType | undefined;
         filterSelector?: FilterSelector;
     }) {
+        SocketFunction.expose(FunctionPreloadController);
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
@@ -655,4 +658,36 @@ export class PathFunctionRunner {
         });
         PathFunctionRunner.RUN_FINISH_COUNT++;
     }
-}
+}
+
+export async function preloadFunctions(specs: FunctionSpec[]) {
+    let nodeIds = await getControllerNodeIdList(FunctionPreloadController);
+    await Promise.allSettled(nodeIds.map(async nodeId => {
+        let controller = FunctionPreloadController.nodes[nodeId.nodeId];
+        console.log(blue(`Preloading functions on ${nodeId}`));
+        await errorToUndefined(controller.preloadFunctions(specs));
+        console.log(blue(`Finished preloading functions on ${nodeId}`));
+    }));
+}
+
+class FunctionPreloaderBase {
+    async preloadFunctions(specs: FunctionSpec[]) {
+        for (let spec of specs) {
+            await getModuleFromSpec(spec);
+        }
+    }
+}
+
+const FunctionPreloadController = SocketFunction.register(
+    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
+    new FunctionPreloaderBase(),
+    () => ({
+        preloadFunctions: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);

package/src/3-path-functions/pathFunctionLoader.ts

@@ -263,7 +263,6 @@ function getCallstackFiles(): string[] {
 async function getModuleFromSpecBase(
     spec: LoadFunctionSpec
 ): Promise<NodeJS.Module> {
-    everLoadedRepos.add(spec.gitURL);
     let hotReloadPackagePath = "";
     let path = gitURLRefMappings.get(getSpecKey(spec));
     let deployPath = "";
@@ -459,37 +458,3 @@ async function executeCommand(command: string, args: string[], options?: {
         });
     });
 }
-
-// Helps us exclude non-function loaders (PathValueServer), which really shouldn't
-//  be loading code anyways!
-let everLoadedRepos = new Set<string>();
-
-export async function preloadFunctions(specs: FunctionSpec[]) {
-    let nodeIds = await getControllerNodeIdList(functionPreloadController);
-    await Promise.all(nodeIds.map(async nodeId => {
-        let controller = functionPreloadController.nodes[nodeId.nodeId];
-        console.log(blue(`Preloading functions on ${nodeId}`));
-        await errorToUndefined(controller.preloadFunctions(specs));
-        console.log(blue(`Finished preloading functions on ${nodeId}`));
-    }));
-}
-
-class FunctionPreloaderBase {
-    async preloadFunctions(specs: FunctionSpec[]) {
-        specs = specs.filter(spec => everLoadedRepos.has(spec.gitURL));
-        for (let spec of specs) {
-            await getModuleFromSpec(spec);
-        }
-    }
-}
-
-const functionPreloadController = SocketFunction.register(
-    "FunctionPreloader-c966a1a6-5d3f-4453-bd03-ae84b574ec00",
-    new FunctionPreloaderBase(),
-    () => ({
-        preloadFunctions: {},
-    }),
-    () => ({
-        hooks: [requiresNetworkTrustHook],
-    })
-);

package/src/4-deploy/deployMain.ts

@@ -1,6 +1,6 @@
 import "../inject";
 
-import { FunctionSpec } from "../3-path-functions/PathFunctionRunner";
+import { FunctionSpec, preloadFunctions } from "../3-path-functions/PathFunctionRunner";
 import path from "path";
 import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "../3-path-functions/syncSchema";
 import { getModuleRelativePath } from "../3-path-functions/syncSchema";
@@ -15,17 +15,20 @@ import yargs from "yargs";
 import { isPublic } from "../config";
 import { deployBlock } from "./deployBlock";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { preloadFunctions } from "../3-path-functions/pathFunctionLoader";
 import { magenta } from "socket-function/src/formatting/logColors";
 import { getGitRef } from "./git";
 import { getGitURL } from "./git";
 import { setLiveDeployedHash } from "./deploySchema";
 import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import debugbreak from "debugbreak";
+import { timeInHour } from "socket-function/src/misc";
+import { preloadUI } from "./edgeNodes";
 
-//todonext;
-// - deployonlycode / deployonlyui / deploynotifyforced
 let yargObj = yargs(process.argv)
     .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
+    .option("deployonlycode", { type: "boolean", desc: "Only deploy code, not ui" })
+    .option("deployonlyui", { type: "boolean", desc: "Only deploy ui, not code" })
+    .option("notifyrefreshdelay", { type: "number", default: timeInHour * 12, desc: "The time clients have to refresh their UI after a deploy." })
     .argv
     ;
 
@@ -116,26 +119,35 @@ export async function deployMain() {
 
     console.log();
     console.log();
-    console.log(magenta(`Preloading ${currentFunctions.length} functions`));
-    await preloadFunctions(currentFunctions);
+    console.log(magenta(`Preloading ${currentFunctions.length} functions & UI`));
+    await Promise.all([
+        preloadFunctions(currentFunctions),
+        preloadUI(gitRef),
+    ]);
 
     console.log();
     console.log();
     console.log(magenta(`Deploying ${currentFunctions.length} functions`));
 
+    let refreshThresholdTime = Date.now() + yargObj.notifyrefreshdelay;
+
+    debugbreak(2);
+    debugger;
     //todonext;
-    // We SHOULD be able to wrap two of the commit functions in one
+    // VERIFY that the nested commit functions run correctly WITH the correct (atomic) flags
     await proxyWatcher.commitFunction({
         debugName: "setLiveDeployedHash",
         inlineNestedWatchers: true,
         watchFunction: () => {
-            void setLiveDeployedHash(gitRef);
-            void replaceFunctions({ domainName, functions: currentFunctions, });
+            if (!yargObj.deployonlycode) {
+                void setLiveDeployedHash({ hash: gitRef, refreshThresholdTime, });
+            }
+            if (!yargObj.deployonlyui) {
+                void replaceFunctions({ domainName, functions: currentFunctions, });
+            }
         },
     });
 
-    await replaceFunctions({ domainName, functions: currentFunctions, });
-
     console.log(magenta(`FINISHED DEPLOY`));
 }
 

package/src/4-deploy/deploySchema.ts

@@ -20,15 +20,21 @@ export const deploySchema = rawSchema<{
 export interface DeployedHash {
     hash: string;
     time: number;
+    refreshThresholdTime: number;
 }
 
 
-export function setLiveDeployedHash(hash: string) {
+export function setLiveDeployedHash(config: {
+    hash: string;
+    refreshThresholdTime: number;
+}) {
+    const { hash, refreshThresholdTime } = config;
     proxyWatcher.writeOnly({
         watchFunction: function writeCall() {
             let hashObj: DeployedHash = {
                 hash,
                 time: Date.now(),
+                refreshThresholdTime,
             };
             deploySchema()[getDomain()].deploy.live = hashObj;
             deploySchema()[getDomain()].deploy.history[nextId()] = hashObj;

package/src/4-deploy/edgeNodes.ts

@@ -16,6 +16,10 @@ import { getGitRef, getGitURL } from "./git";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import path from "path";
 import debugbreak from "debugbreak";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getControllerNodeIdList } from "../-g-core-values/NodeCapabilities";
+import { blue } from "socket-function/src/formatting/logColors";
+import { errorToUndefined } from "../errors";
 
 const UPDATE_POLL_INTERVAL = timeInMinute;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -51,6 +55,7 @@ export type EdgeNodesIndex = {
     edgeNodes: EdgeNodeConfig[];
 };
 
+
 // IMPORTANT! The EdgeNodeConfig MUST be immutable, otherwise every node has to read every EdgeNodeConfig
 //  every poll, which is too much reading.
 export type EdgeNodeConfig = {
@@ -127,24 +132,6 @@ async function loadEntryPointsByHash(config: {
         FunctionId: "entryPoint",
     });
 
-    //todonext;
-    //  3) Setup the deployMain code to set the live hash
-    //      - Flags to decide what to set
-    //      - Inside of replaceFunctions, via passing a callback function to it. The nested commits SHOULD
-    //          just work?
-    //          - IMPORTANT! Step through the code, to ensure the nested flags are set correctly. If not...
-    //              update the proxy watcher to set them properly...
-    //      - With all the flags (that only do one or the other)
-    //      - Add preloading capability, so we can preload the hashes before we deploy
-    //  3) Dynamically watch the live hash, and load new hashes (deploying their services)
-    //  - Test by pushing, deploying, and refreshing (we should prefer to latest service)
-    //  3.1) If local, DO NOT watch the live hash
-    //  5) Write the live hash to public backblaze, and have the client read it, and prefer that hash
-    //  6) Pass the live hash to the client with the edge node
-    //  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
-    //  - Test by pushing, deploying, and then we should immediately see a notify to refresh
-
-
     let depth = entryPaths[0].replaceAll("\\", "/").split("/").filter(x => x && x !== ".").length;
 
     let firstPath = path.resolve(module.filename);
@@ -167,6 +154,8 @@ async function loadEntryPointsByHash(config: {
     registeredEdgeNode = edgeNodeConfig;
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
+
+    SocketFunction.expose(EdgeNodeController);
 }
 
 export const getEdgeNodeConfigURL = lazy(async () => {
@@ -244,6 +233,46 @@ async function updateEdgeNodesFile() {
     await edgeNodeStorage.set("edge-nodes-index.json", Buffer.from(JSON.stringify(newEdgeNodeIndex)));
 }
 
+export async function preloadUI(hash: string) {
+    let nodeIds = await getControllerNodeIdList(EdgeNodeController);
+    await Promise.allSettled(nodeIds.map(async nodeId => {
+        let controller = EdgeNodeController.nodes[nodeId.nodeId];
+        console.log(blue(`Preloading functions on ${nodeId}`));
+        await errorToUndefined(controller.preloadUI({ hash }));
+        console.log(blue(`Finished preloading functions on ${nodeId}`));
+    }));
+}
+
+class EdgeNodeControllerBase {
+    public async preloadUI(config: {
+        hash: string;
+    }) {
+        const { hash } = config;
+        if (!registeredEdgeNode || !canHaveChildren(registeredEdgeNode)) {
+            throw new Error(`Somehow we have no registered edge node. How did we even expose this controller?`);
+        }
+        await loadEntryPointsByHash({
+            host: registeredEdgeNode.host,
+            entryPaths: registeredEdgeNode.entryPaths,
+            hash,
+        });
+    }
+}
+
+const EdgeNodeController = SocketFunction.register(
+    "EdgeNodeController-e0110af9-f8c4-42f3-ad11-633f8e43abdd",
+    new EdgeNodeControllerBase(),
+    () => ({
+        preloadUI: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+
 // async function main() {
 //     await loadEntryPointsByHash({
 //         host: "127-0-0-1.querysub.com:1111",
@@ -251,4 +280,26 @@ async function updateEdgeNodesFile() {
 //         hash: "7b1eb4934d5bdfe6d0f7076049c4b7decad4e645",
 //     });
 // }
-// main().catch(console.error).finally(() => process.exit());
+// main().catch(console.error).finally(() => process.exit());
+
+
+
+//todonext;
+//  3) Setup the deployMain code to set the live hash
+//      - Flags to decide what to set
+//      - Inside of replaceFunctions, via passing a callback function to it. The nested commits SHOULD
+//          just work?
+//          - IMPORTANT! Step through the code, to ensure the nested flags are set correctly. If not...
+//              update the proxy watcher to set them properly...
+//      - With all the flags (that only do one or the other)
+//      - Add preloading capability, so we can preload the hashes before we deploy
+
+//  3) Dynamically watch the live hash, and load new hashes (deploying their services)
+//  - Test by pushing, deploying, and refreshing (we should prefer to latest service)
+//  3.1) If local, DO NOT watch the live hash
+
+//  5) Write the live hash to public backblaze, and have the client read it, and prefer that hash
+//  6) Pass the live hash to the client with the edge node
+//  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
+//  - Test by pushing, deploying, and then we should immediately see a notify to refresh
+//  8) VERIFY writing and permissions stillw orks

package/src/4-querysub/permissions.ts

@@ -90,8 +90,6 @@ export class PermissionsCheck {
     }>();
     // IMPORTANT! This function USED TO BE a major hotspot for function evaluation, and so is heavily optimized.
     public checkPermissions(path: string): { permissionsPath: string; allowed: boolean; } {
-        //todonext
-        //return { permissionsPath: path, allowed: true };
         if (this.dead) throw new Error("PermissionsCheck MUST be used synchronously, after which it cannot be reused");
         if (PermissionsCheck.skippingChecks) return { permissionsPath: rootPathStr, allowed: true };