querysub 0.43.0 → 0.45.0

package/package.json

@@ -1,60 +1,60 @@
-{
-    "name": "querysub",
-    "version": "0.43.0",
-    "main": "index.js",
-    "license": "MIT",
-    "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/sliftist/shard.git"
-    },
-    "dependencies": {
-        "@sendgrid/mail": "^7.7.0",
-        "@types/fs-ext": "^2.0.3",
-        "@types/node-forge": "^1.3.1",
-        "@types/pako": "^2.0.3",
-        "@types/yargs": "^15.0.5",
-        "acme-client": "^5.0.0",
-        "cbor-x": "^1.6.0",
-        "chalk": "^5.2.0",
-        "diskusage": "^1.2.0",
-        "fs-ext": "^2.0.0",
-        "js-sha256": "https://github.com/sliftist/js-sha256",
-        "js-sha512": "^0.9.0",
-        "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
-        "pako": "^2.1.0",
-        "preact": "^10.11.3",
-        "socket-function": "^0.39.0",
-        "terser": "^5.31.0",
-        "typesafecss": "^0.6.3",
-        "yaml": "^2.5.0",
-        "yargs": "^15.3.1"
-    },
-    "optionalDependencies": {
-        "rdtsc-now": "^0.4.2"
-    },
-    "scripts": {
-        "server": "yarn typenode ./src/server.ts",
-        "serversharded": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
-        "function": "yarn typenode ./src/3-path-functions/PathFunctionRunnerMain.ts --function --local D:/repos/shard/",
-        "deploy": "yarn typenode ./src/3-path-functions/deployMain.ts --domain querysub.com --local D:/repos/shard/",
-        "type": "yarn tsc --noEmit",
-        "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-        "dbmerge": "yarn typenode ./src/dbmerge.ts",
-        "test": "yarn typenode ./src/test/test.tsx --local D:/repos/shard/",
-        "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local D:/repos/shard/"
-    },
-    "bin": {
-        "querysub-deploy": "./bin/deploy.js",
-        "querysub-server": "./bin/server.js",
-        "querysub-function": "./bin/function.js"
-    },
-    "devDependencies": {
-        "dependency-cruiser": "^12.11.0",
-        "open": "^8.4.0",
-        "typedev": "^0.3.0"
-    },
-    "resolutions": {
-        "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6"
-    }
-}
+{
+  "name": "querysub",
+  "version": "0.45.0",
+  "main": "index.js",
+  "license": "MIT",
+  "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sliftist/shard.git"
+  },
+  "dependencies": {
+    "@sendgrid/mail": "^7.7.0",
+    "@types/fs-ext": "^2.0.3",
+    "@types/node-forge": "^1.3.1",
+    "@types/pako": "^2.0.3",
+    "@types/yargs": "^15.0.5",
+    "acme-client": "^5.0.0",
+    "cbor-x": "^1.6.0",
+    "chalk": "^5.2.0",
+    "diskusage": "^1.2.0",
+    "fs-ext": "^2.0.0",
+    "js-sha256": "https://github.com/sliftist/js-sha256",
+    "js-sha512": "^0.9.0",
+    "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
+    "pako": "^2.1.0",
+    "preact": "^10.11.3",
+    "socket-function": "^0.41.0",
+    "terser": "^5.31.0",
+    "typesafecss": "^0.6.3",
+    "yaml": "^2.5.0",
+    "yargs": "^15.3.1"
+  },
+  "optionalDependencies": {
+    "rdtsc-now": "^0.4.2"
+  },
+  "scripts": {
+    "server": "yarn typenode ./src/server.ts",
+    "serversharded": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
+    "function": "yarn typenode ./src/3-path-functions/PathFunctionRunnerMain.ts --function --local D:/repos/shard/",
+    "deploy": "yarn typenode ./src/3-path-functions/deployMain.ts --domain querysub.com --local D:/repos/shard/",
+    "type": "yarn tsc --noEmit",
+    "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
+    "dbmerge": "yarn typenode ./src/dbmerge.ts",
+    "test": "yarn typenode ./src/test/test.tsx --local D:/repos/shard/",
+    "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local D:/repos/shard/"
+  },
+  "bin": {
+    "querysub-deploy": "./bin/deploy.js",
+    "querysub-server": "./bin/server.js",
+    "querysub-function": "./bin/function.js"
+  },
+  "devDependencies": {
+    "dependency-cruiser": "^12.11.0",
+    "open": "^8.4.0",
+    "typedev": "^0.3.0"
+  },
+  "resolutions": {
+    "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6"
+  }
+}

package/src/-a-archives/archivesBackBlaze.ts

@@ -10,6 +10,7 @@ import { devDebugbreak } from "../config";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green } from "socket-function/src/formatting/logColors";
 import debugbreak from "debugbreak";
+import { addTimeProfileDistribution, onTimeProfile } from "../5-diagnostics/nodeMetadata";
 
 export function hasBackblazePermissions() {
     return isNode() && fs.existsSync(getBackblazePath());
@@ -68,11 +69,13 @@ const getAPI = lazy(async () => {
             }
             try {
                 let url = auth.apiUrl + "/b2api/v2/" + name;
+                let time = Date.now();
                 let result = await httpsRequest(url, Buffer.from(JSON.stringify(arg)), type, undefined, {
                     headers: {
                         Authorization: auth.authorizationToken,
                     }
                 });
+                onTimeProfile("Backblaze API", time);
                 return JSON.parse(result.toString());
             } catch (e: any) {
                 throw new Error(`Error in ${name}, arg ${JSON.stringify(arg).slice(0, 1000)}: ${e.stack}`);

package/src/-g-core-values/NodeCapabilities.ts

@@ -51,17 +51,19 @@ export async function getControllerNodeId(
 
 export async function getControllerNodeIdList(
     controller: SocketRegistered<{}>,
-): Promise<string[]> {
+): Promise<{ nodeId: string; entryPoint: string }[]> {
     let nodeIdsToTest = await getAllNodeIds();
-    let passedNodeIds = new Set<string>();
+    let passedNodeIds = new Map<string, string>();
     await Promise.all(nodeIdsToTest.map(async nodeId => {
         if (await doesNodeExposeController(nodeId, controller)) {
-            passedNodeIds.add(nodeId);
+            let entryPoint = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[nodeId].getEntryPoint()) || "Entry point timedout";
+            passedNodeIds.set(nodeId, entryPoint);
         }
     }));
-    return Array.from(passedNodeIds);
+    return Array.from(passedNodeIds.entries()).map(([nodeId, entryPoint]) => ({ nodeId, entryPoint }));
 }
 
+
 export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
     let exposedControllers = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getExposedControllers());
     return !!exposedControllers?.includes(controller._classGuid);

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -31,6 +31,8 @@ const MAX_RECONNECT_TIME = timeInMinute * 15;
 const RECONNECT_POLL_INTERVAL = 10000;
 
 // Poll nodes that appear dead. Without this, if the internet goes down, we might forever ignore nodes.
+const INITIAL_RECOVERY_POLL_INTERVAL = timeInSecond * 5;
+const INITIAL_RECOVERY_RUNS = 100;
 const RECOVERY_POLL_INTERVAL = timeInMinute;
 
 // NOTE: We don't implicitly include parents, as when we add multiple specific paths it is easier
@@ -190,6 +192,8 @@ class NodePathAuthorities {
         return this.isReadReady;
     }
 
+    private previouslyNotAvailableNodes = new Set<string>();
+
     @measureFnc
     private async watchAuthorityPaths() {
         await onNodeDiscoveryReady();
@@ -237,10 +241,15 @@ class NodePathAuthorities {
                 if (createTime === undefined) {
                     // Don't log for 127-0-0-1, as it usually fails, and is mostly a development optimization
                     if (!nodeId.includes("127-0-0-1")) {
-                        diskLog(`Node didn't respond to getCreateTime`, { nodeId });
+                        if (!this.previouslyNotAvailableNodes.has(nodeId)) {
+                            console.log(yellow(`Node didn't respond to getCreateTime`), { nodeId });
+                        }
                     }
+                    this.previouslyNotAvailableNodes.add(nodeId);
                     return;
                 }
+                this.previouslyNotAvailableNodes.delete(nodeId);
+
                 time = Date.now() - time;
                 console.log(blue(`Identifying ${nodeId} as a path authority. ping latency = ${formatTime(time)}`));
                 this.authorities.set(nodeId, {
@@ -294,7 +303,20 @@ class NodePathAuthorities {
         watchDeltaNodeIds(obj => ingestNewNodeIds(obj.newNodeIds, obj.removedNodeIds));
         await firstPromise.promise;
 
-        runInfinitePoll(RECOVERY_POLL_INTERVAL, async () => {
+        let recoveryIndex = 0;
+        runInfinitePoll(INITIAL_RECOVERY_POLL_INTERVAL, async () => {
+            // Wait longer the more times we loop. This allows us to reconnect quickly on startup
+            //  (when other nodes might be unavailable or in a weird state), but not waste money
+            //  hitting backblaze constantly, when we're already in a stable state.
+            {
+                recoveryIndex++;
+                let f = Math.min(1, recoveryIndex / INITIAL_RECOVERY_RUNS);
+                let delayTime = (1 - f) * INITIAL_RECOVERY_POLL_INTERVAL + f * RECOVERY_POLL_INTERVAL;
+                if (delayTime > INITIAL_RECOVERY_POLL_INTERVAL) {
+                    await delay(delayTime - INITIAL_RECOVERY_POLL_INTERVAL);
+                }
+            }
+
             let nodes = await getAllNodeIds();
             let nonExistentNodes: string[] = [];
             for (let node of nodes) {