querysub 0.115.0 → 0.117.0

package/bin/deploy.js

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
 
 require("typenode");
-require("../src/3-path-functions/deployMain");
+require("../src/4-deploy/deployMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.115.0",
+  "version": "0.117.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "preact": "^10.11.3",
-    "socket-function": "^0.77.0",
+    "socket-function": "^0.78.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.6.3",
     "yaml": "^2.5.0",
@@ -37,7 +37,7 @@
     "server": "yarn typenode ./src/server.ts",
     "serversharded": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
     "function": "yarn typenode ./src/3-path-functions/PathFunctionRunnerMain.ts --function --local D:/repos/shard/",
-    "deploy": "yarn typenode ./src/3-path-functions/deployMain.ts --domain querysub.com --local D:/repos/shard/",
+    "deploy": "yarn typenode ./src/4-deploy/deployMain.ts --domain querysub.com --local D:/repos/shard/",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
     "dbmerge": "yarn typenode ./src/dbmerge.ts",
@@ -57,4 +57,4 @@
   "resolutions": {
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6"
   }
-}
+}

package/src/-a-archives/archivesBackBlaze.ts

@@ -406,7 +406,7 @@ export class ArchivesBackblaze {
             corsRuleName: "onlyCurrentOrigin",
             allowedOrigins: ["*"],
             allowedOperations: ["b2_download_file_by_id", "b2_download_file_by_name"],
-            allowedHeaders: [],
+            allowedHeaders: ["range"],
             exposeHeaders: ["x-bz-content-sha1"],
             maxAgeSeconds: cacheTime / 1000,
         }] : [];

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -23,7 +23,7 @@ import dns from "dns/promises";
 import { isDefined } from "../misc";
 import { diskLog, noDiskLogPrefix } from "../diagnostics/logs/diskLogger";
 import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
-import { getBootedEdgeNode } from "../4-querysub/edge/edgeBootstrap";
+import { getBootedEdgeNode } from "../4-deploy/edgeBootstrap";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 2;
 // Interval which we check other heartbeats

package/src/3-path-functions/PathFunctionHelpers.ts

@@ -15,8 +15,6 @@ import { getPathStr2 } from "../path";
 import { isNode, sort } from "socket-function/src/misc";
 import { decodeCborx, encodeCborx } from "../misc/cloneHelpers";
 import { parseFilterable } from "../misc/filterable";
-const cborxInstance = lazy(() => new cborx.Encoder({ structuredClone: true }));
-
 
 // NOTE: We could deploy single functions, but... we will almost always be updating all functions at
 //      once, because keeping everything on the same git hash reduces a lot of potential bugs.

package/src/3-path-functions/pathFunctionLoader.ts

@@ -24,6 +24,13 @@ import { sha256 } from "js-sha256";
 import os from "os";
 import { formatTime } from "socket-function/src/formatting/format";
 
+export type LoadFunctionSpec = {
+    gitURL: string;
+    gitRef: string;
+    FilePath: string;
+    FunctionId: string;
+};
+
 // Get localPathRemappings using yargs, so it is easy to configure in multiple entry points
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("local", { type: "boolean", desc: `If true, uses the local directory instead of the remote git repo. Also hotreloads from disk. Determines the repo to replace through the package.json "repository" property.` })
@@ -82,20 +89,17 @@ let moduleCache = new Map<string, {
     result: MaybePromise<NodeJS.Module>;
     error: Error | undefined;
 }>();
-function getSpecKey(spec: FunctionSpec) {
+function getSpecKey(spec: LoadFunctionSpec) {
     // Only include these specific keys AND order them, so the hash hits no matter how this function is called
     spec = {
         FilePath: spec.FilePath,
         gitURL: spec.gitURL,
         gitRef: spec.gitRef,
-        DomainName: spec.DomainName,
-        exportPathStr: spec.exportPathStr,
         FunctionId: spec.FunctionId,
-        ModuleId: spec.ModuleId,
     };
     return JSON.stringify(spec);
 }
-export function getModuleFromConfig(spec: FunctionSpec): MaybePromise<NodeJS.Module> {
+export function getModuleFromConfig(spec: LoadFunctionSpec): MaybePromise<NodeJS.Module> {
     let key = getSpecKey(spec);
     let value = moduleCache.get(key);
     if (!value) {
@@ -126,7 +130,10 @@ export function setGitURLMapping(config: {
 const loadTimeIndicatorFileName = "loadTimeIndicator-71cd93ba-1667-49ac-9206-b27930bbd983";
 
 /** spec => path that we can use with require */
-let moduleResolver = async (spec: FunctionSpec) => {
+let moduleResolver = async (spec: {
+    gitURL: string;
+    gitRef: string;
+}) => {
     let gitURL = spec.gitURL;
     let urlForPath = gitURL;
 
@@ -160,7 +167,7 @@ let moduleResolver = async (spec: FunctionSpec) => {
 
             // Remove any previous attempt to sync it
             if (fs.existsSync(repoPath)) {
-                await fs.promises.rename(repoPath, repoPath + "_" + Date.now());
+                await fs.promises.rename(repoPath, repoPath.slice(0, -1) + "_" + Date.now());
             }
             // Clone it
             await executeCommand("git", ["clone", gitURL, repoPath]);
@@ -257,7 +264,7 @@ function getCallstackFiles(): string[] {
 }
 
 async function getModuleFromSpecBase(
-    spec: FunctionSpec
+    spec: LoadFunctionSpec
 ): Promise<NodeJS.Module> {
     everLoadedRepos.add(spec.gitURL);
     let hotReloadPackagePath = "";
@@ -328,10 +335,10 @@ async function getModuleFromSpecBase(
     return module;
 }
 
-let evaluatingFunctionSpec: FunctionSpec | undefined;
+let evaluatingFunctionSpec: LoadFunctionSpec | undefined;
 
-let moduleToSpec = new Map<NodeJS.Module, FunctionSpec>();
-export function getSpecFromModule(module: NodeJS.Module): FunctionSpec | undefined {
+let moduleToSpec = new Map<NodeJS.Module, LoadFunctionSpec>();
+export function getSpecFromModule(module: NodeJS.Module): LoadFunctionSpec | undefined {
     return evaluatingFunctionSpec ?? moduleToSpec.get(module);
 }
 
@@ -401,16 +408,59 @@ const hotreloadIfChanged = batchFunction({ delay: 100, name: "hotreloadIfChanged
     }
 });
 
+async function which(command: string): Promise<string> {
+    let whichOrWhere = os.platform() === "win32" ? "where" : "which";
+    let path = child_process.execSync(`${whichOrWhere} ${command}`).toString().trim().replaceAll("\r", "").split("\n")[0].trim().replaceAll("\\", "/");
+    if (!path) {
+        throw new Error(`Command ${command} not found`);
+    }
+    path = `"${path}"`;
+    return path;
+}
+
 async function executeCommand(command: string, args: string[], options?: {
     cwd?: string;
-}) {
-    child_process.execSync(`${command} ${args.join(" ")}`, {
-        stdio: "inherit",
-        cwd: options?.cwd
+}): Promise<string> {
+    let resolvedCommandPath = await which(command);
+    let debug = `${resolvedCommandPath} ${args.join(" ")}`;
+    if (options?.cwd) {
+        debug += ` (in ${options.cwd})`;
+    }
+    console.log(`Running command: ${debug}`);
+    return new Promise((resolve, reject) => {
+        const child = child_process.spawn(resolvedCommandPath, args, {
+            cwd: options?.cwd,
+            stdio: ["ignore", "pipe", "pipe"],
+            shell: process.platform === "win32",
+        });
+
+        let stdout = "";
+        let stderr = "";
+
+        child.stdout.on("data", (data) => {
+            console.log(data.toString());
+            stdout += data.toString();
+        });
+
+        child.stderr.on("data", (data) => {
+            console.log(data.toString());
+            stderr += data.toString();
+        });
+
+        child.on("error", (error) => {
+            reject(new Error(`Failed to execute command ${debug}: ${error.message}`));
+        });
+
+        child.on("close", (code) => {
+            if (code !== 0) {
+                reject(new Error(`Command ${debug} failed with code ${code}.\nstdout: ${JSON.stringify(stdout)}\nstderr: ${JSON.stringify(stderr)}`));
+                return;
+            }
+            resolve(stdout.trim());
+        });
     });
 }
 
-
 // Helps us exclude non-function loaders (PathValueServer), which really shouldn't
 //  be loading code anyways!
 let everLoadedRepos = new Set<string>();

package/src/{3-path-functions → 4-deploy}/deployMain.ts

@@ -1,14 +1,13 @@
 import "../inject";
 
-import { FunctionSpec } from "./PathFunctionRunner";
+import { FunctionSpec } from "../3-path-functions/PathFunctionRunner";
 import path from "path";
-import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "./syncSchema";
-import { getModuleRelativePath } from "./syncSchema";
-import * as child_process from "child_process";
+import { getExportPath, getSchemaObject, PERMISSIONS_FUNCTION_ID } from "../3-path-functions/syncSchema";
+import { getModuleRelativePath } from "../3-path-functions/syncSchema";
 import fs from "fs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "../-a-auth/certs";
-import { replaceFunctions } from "./PathFunctionHelpers";
+import { replaceFunctions } from "../3-path-functions/PathFunctionHelpers";
 import { getPathStr2 } from "../path";
 import { setIsDeploy } from "./deployCheck";
 
@@ -16,11 +15,13 @@ import yargs from "yargs";
 import { isPublic } from "../config";
 import { deployBlock } from "./deployBlock";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
-import { preloadFunctions } from "./pathFunctionLoader";
+import { preloadFunctions } from "../3-path-functions/pathFunctionLoader";
 import { magenta } from "socket-function/src/formatting/logColors";
+import { getGitRef } from "./git";
+import { getGitURL } from "./git";
+import { setLiveDeployedHash } from "./deploySchema";
 let yargObj = yargs(process.argv)
     .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
-    .option("nogit", { type: "boolean", default: false, desc: `Disable git operations. Useful for debugging, when using --local` })
     .argv
     ;
 
@@ -56,6 +57,12 @@ export async function deployMain() {
     }
     let gitRef = getGitRef(gitDir);
 
+
+    //todonext
+    // Flags to set this
+    //setLiveDeployedHash(gitRef);
+
+
     let usedModules = new Map<string, string>();
 
     for (let module of Object.values(require.cache)) {
@@ -121,13 +128,6 @@ export async function deployMain() {
 
     console.log(magenta(`FINISHED DEPLOY`));
 }
-function getGitURL(gitDir: string) {
-    if (yargObj.nogit) return "git@github.com:nogit/nogit.git";
-    return child_process.execSync(`git remote get-url origin`, { cwd: gitDir }).toString().trim();
-}
-function getGitRef(gitDir: string) {
-    if (yargObj.nogit) return "nogit";
-    return child_process.execSync(`git rev-parse HEAD`, { cwd: gitDir }).toString().trim();
-}
+
 
 deployMain().catch(e => console.error(e)).finally(() => process.exit());

package/src/4-deploy/deploySchema.ts

@@ -0,0 +1,69 @@
+import { nextId } from "socket-function/src/misc";
+import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
+import { functionSchema } from "../3-path-functions/PathFunctionRunner";
+import { syncSchema } from "../3-path-functions/syncSchema";
+import { getDomain } from "../config";
+
+//todonext
+//  Current hash, history of hashes
+export const deploySchema = rawSchema<{
+    [domainName: string]: {
+        deploy: {
+            live: DeployedHash;
+            history: {
+                [id: string]: DeployedHash;
+            };
+        };
+    };
+}>();
+export interface DeployedHash {
+    hash: string;
+    time: number;
+}
+
+
+export function setLiveDeployedHash(hash: string) {
+    proxyWatcher.writeOnly({
+        watchFunction: function writeCall() {
+            let hashObj: DeployedHash = {
+                hash,
+                time: Date.now(),
+            };
+            deploySchema()[getDomain()].deploy.live = hashObj;
+            deploySchema()[getDomain()].deploy.history[nextId()] = hashObj;
+        },
+    });
+}
+
+//todonext
+//  - Function to dynamically deploy edge service based on hash (try on old hashes)
+//  - Watch live hash, and when it changes, deploy new edge service
+//      - Add flag to deploy script which sets it (with all the flags, so we can just deploy it)
+//  - Update edge node code to pass live hash, and try to connect to a service with that
+
+//todonext
+/*
+Force client refresh mechanism / update notification ON `yarn deploy`
+    - deployonlycode / deployonlyui / deploynotifyforced
+    - OH! Actually...
+    - This HAS to use a synced value!
+    - Add a new synced value which specifies the UI hash!
+        - THEN! Have front-end servers dynamically launch this services with this hash when it's set (loading the code and then serving the specific path, with the hash, etc)
+        - Also, serve this to clients!
+    - Clients will watch this value (normally, like any other value), and notify the user when it is out of date, letting them ignore it, but eventually forcing a refresh
+    - AND, support preloading it
+    - So our order will be
+        (git push)
+        preload functions
+        preload UI
+        atomically set function hashes + ui hashes
+        clients are notified they should refresh
+        new clients use services with the latest hash (which servers are already hosting)
+        functions will run this hash
+    - Ah, also... add a flag which tells us NOT to use the service HTML cache
+        - Remove it when used
+        - Required on refresh, otherwise we might refresh and use an outdated service
+        
+ALSO, have FunctionRunner watch the hash using the get/watch mechanism (for preloading, as it will dynamically load the code depending on what is really deployed anyways...)
+*/

package/src/{4-querysub/edge → 4-deploy}/edgeBootstrap.ts

@@ -1,5 +1,5 @@
 import { cache } from "socket-function/src/caching";
-import { isServer } from "../../config2";
+import { isServer } from "../config2";
 import { EdgeNodeConfig, EdgeNodesIndex } from "./edgeNodes";
 import { timeInMinute } from "socket-function/src/misc";
 import { measureBlock } from "socket-function/src/profiling/measure";
@@ -279,7 +279,6 @@ async function edgeNodeFunction(config: {
             return true;
         });
 
-
         // If they are using an IP domain, that's the only node we accept (we still need to pick the node
         //  though, as we need to know the entryPaths to import)
         {

package/src/{4-querysub/edge → 4-deploy}/edgeNodes.ts

@@ -1,17 +1,21 @@
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
-import { getOwnMachineId } from "../../-a-auth/certs";
-import { getDomain, isPublic } from "../../config";
+import { getOwnMachineId } from "../-a-auth/certs";
+import { devDebugbreak, getDomain, isPublic } from "../config";
 import child_process from "child_process";
-import { getAllNodeIds, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
-import { getArchivesBackblazePublic } from "../../-a-archives/archivesBackBlaze";
-import { nestArchives } from "../../-a-archives/archives";
+import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
+import { nestArchives } from "../-a-archives/archives";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { compare, compareArray, timeInMinute } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
-import { shutdown } from "../../diagnostics/periodic";
-import { hostArchives } from "../../-b-authorities/cdnAuthority";
+import { shutdown } from "../diagnostics/periodic";
+import { hostArchives } from "../-b-authorities/cdnAuthority";
+import { getGitRef, getGitURL } from "./git";
+import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
+import path from "path";
+import debugbreak from "debugbreak";
 
 const UPDATE_POLL_INTERVAL = timeInMinute;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -70,7 +74,8 @@ export async function registerEdgeNode(config: {
 
     await waitForFirstTimeSync();
 
-    let gitHash = child_process.execSync("git rev-parse HEAD").toString().trim();
+
+    let gitHash = getGitRef();
     let nodeId = getOwnNodeId();
     let machineId = getOwnMachineId();
 
@@ -87,9 +92,83 @@ export async function registerEdgeNode(config: {
 
     await edgeNodeStorage.set("node_" + nodeId, Buffer.from(JSON.stringify(edgeNodeConfig)));
 
+    await loadEntryPointsByHash({
+        host: config.host,
+        entryPaths: config.entryPaths,
+        hash: gitHash,
+    });
+
     await startUpdateLoop();
 }
 
+async function loadEntryPointsByHash(config: {
+    host: string;
+    entryPaths: string[];
+    hash: string;
+}) {
+    let gitHash = config.hash;
+    let entryPaths = config.entryPaths;
+
+
+    debugbreak(2);
+    debugger;
+
+    let module = await getModuleFromConfig({
+        gitURL: getGitURL(),
+        gitRef: gitHash,
+        FilePath: entryPaths[0],
+        FunctionId: "entryPoint",
+    });
+
+
+    debugbreak(2);
+    debugger;
+
+    module.filename;
+    debugger;
+
+    //todonext;
+    //  1) Run this file specifically, and verify this code somewhat runs
+    //  2) Run aipaint normally. This should have a later bootTime, and therefore be picked,
+    //      so it should test our browser loading other gitHashes
+    //  3) Setup the deployMain code to set the live hash
+    //      - Inside of replaceFunctions, via passing a callback function to it. The nested commits SHOULD
+    //          just work? Hmm... I guess verify they do (that the writeOnly functions, etc)
+    //      - With all the flags (that only do one or the other)
+    //      - Add preloading capability, so we can preload the hashes before we deploy
+    //  3) Dynamically watch the live hash, and load new hashes (deploying their services)
+    //  - Test by pushing, deploying, and refreshing (we should prefer to latest service)
+    //  3.1) If local, DO NOT watch the live hash
+    //  5) Write the live hash to public backblaze, and have the client read it, and prefer that hash
+    //  6) Pass the live hash to the client with the edge node
+    //  7) Client live hash watch code + notify + refresh + flag to ignore the baked in live hash + edge node
+    //  - Test by pushing, deploying, and then we should immediately see a notify to refresh
+
+
+    let depth = entryPaths[0].replaceAll("\\", "/").split("/").filter(x => x && x !== ".").length;
+
+    let firstPath = path.resolve(module.filename);
+    let rootPath = firstPath.replaceAll("\\", "/").split("/").slice(0, -depth).join("/") + "/";
+
+    entryPaths = entryPaths.map(x => path.resolve(rootPath + x).replaceAll("\\", "/"));
+
+    let nodeId = getOwnNodeId();
+    let machineId = getOwnMachineId();
+
+    let edgeNodeConfig: EdgeNodeConfig = {
+        nodeId,
+        machineId,
+        host: config.host,
+        gitHash,
+        bootTime: Date.now(),
+        entryPaths,
+        public: isPublic(),
+    };
+    registeredEdgeNode = edgeNodeConfig;
+
+    await edgeNodeStorage.set("node_" + nodeId, Buffer.from(JSON.stringify(edgeNodeConfig)));
+}
+
 export const getEdgeNodeConfigURL = lazy(async () => {
     let { getURL } = await hostArchives({
         archives: edgeNodeStorage,
@@ -163,4 +242,13 @@ async function updateEdgeNodesFile() {
     }
 
     await edgeNodeStorage.set("edge-nodes-index.json", Buffer.from(JSON.stringify(newEdgeNodeIndex)));
-}
+}
+
+async function main() {
+    await loadEntryPointsByHash({
+        host: "127-0-0-1.querysub.com:1111",
+        entryPaths: ["./src/browser.ts"],
+        hash: "2d4fe9f2d1c80b7edd4c0f0ec4842fc38eab3b45",
+    });
+}
+main().catch(console.error).finally(() => process.exit());

package/src/4-deploy/git.ts

@@ -0,0 +1,7 @@
+import * as child_process from "child_process";
+export function getGitURL(gitDir = ".") {
+    return child_process.execSync(`git remote get-url origin`, { cwd: gitDir }).toString().trim();
+}
+export function getGitRef(gitDir = ".") {
+    return child_process.execSync(`git rev-parse HEAD`, { cwd: gitDir }).toString().trim();
+}

package/src/4-dom/qreact.tsx

@@ -1,6 +1,6 @@
 import type preact from "preact";
 import { isNode, sort } from "socket-function/src/misc";
-import { isDeploy } from "../3-path-functions/deployCheck";
+import { isDeploy } from "../4-deploy/deployCheck";
 import { PermissionsCheck } from "../4-querysub/permissions";
 import { ClientWatcher, clientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";

package/src/4-dom/qreactTest.tsx

@@ -4,7 +4,7 @@ disableMeasurements();
 import preact from "preact";
 import { isNode, list, timeInHour } from "socket-function/src/misc";
 import { ClientWatcher, clientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { isDeploy } from "../3-path-functions/deployCheck";
+import { isDeploy } from "../4-deploy/deployCheck";
 import { logErrors } from "../errors";
 import { watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadController";
 import { qreact } from "./qreact";

package/src/4-querysub/Querysub.ts

@@ -891,5 +891,5 @@ registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
 import "../diagnostics/watchdog";
 import "../diagnostics/trackResources";
 import "../diagnostics/benchmark";
-import { getEdgeNodeConfigURL, registerEdgeNode } from "./edge/edgeNodes"; import { getEdgeBootstrapScript } from "./edge/edgeBootstrap";
+import { getEdgeNodeConfigURL, registerEdgeNode } from "../4-deploy/edgeNodes"; import { getEdgeBootstrapScript } from "../4-deploy/edgeBootstrap";
 

package/src/test/test.tsx

@@ -7,7 +7,7 @@ import { watchFilesAndTriggerHotReloading } from "socket-function/hot/HotReloadC
 import { Querysub } from "../4-querysub/QuerysubController";
 import { red } from "socket-function/src/formatting/logColors";
 import { PermissionsCheck } from "../4-querysub/permissions";
-import { isDeploy } from "../3-path-functions/deployCheck";
+import { isDeploy } from "../4-deploy/deployCheck";
 import { formatTime } from "socket-function/src/formatting/format";
 
 import { PromiseObj } from "../promise";