querysub 0.391.0 → 0.392.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.391.0",
+  "version": "0.392.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -57,7 +57,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "querysub": "^0.357.0",
-    "socket-function": "^1.1.2",
+    "socket-function": "^1.1.4",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -2,12 +2,12 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { errorToUndefined, errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
 import { PromiseObj } from "../promise";
-import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, isNodeDiscoveryLogging, isOwnNodeId, onNodeDiscoveryReady, triggerNodeChange, watchDeltaNodeIds, watchNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, isNodeDiscoveryLogging, isNodeIdLocal, isOwnNodeId, onNodeDiscoveryReady, triggerNodeChange, watchDeltaNodeIds, watchNodeIds } from "../-f-node-discovery/NodeDiscovery";
 import { PathValueController } from "./PathValueController";
 import { MAX_ACCEPTED_AUTHORITY_STARTUP_TIME, PathValueSnapshot, STARTUP_CUTOFF_TIME, authorityStorage, matchesParentRangeFilterPart } from "./pathValueCore";
 import { pathValueArchives } from "./pathValueArchives";
 import { deepCloneJSON, isNode, sha256Hash, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { delay, runInfinitePoll } from "socket-function/src/batching";
+import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
 import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
 import debugbreak from "debugbreak";
 import { getNodeIdFromLocation, getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
@@ -22,6 +22,7 @@ import { IdentityController_getCurrentReconnectNodeIdAssert, IdentityController_
 import { getBufferFraction, getBufferInt, getShortNumber } from "../bits";
 import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
+import { testTCPIsListening } from "socket-function/src/networking";
 
 export const LOCAL_DOMAIN = "LOCAL";
 export const LOCAL_DOMAIN_PATH = getPathStr1(LOCAL_DOMAIN);
@@ -36,6 +37,7 @@ const MAX_RECONNECT_TIME = timeInMinute * 15;
 const RECONNECT_POLL_INTERVAL = 10000;
 
 // Poll nodes that appear dead. Without this, if the internet goes down, we might forever ignore nodes.
+// NOTE: During development this might seem excessive, however, after about an hour this should ramp up to the full poll interval of a minute, which should be fairly reasonable.
 const INITIAL_RECOVERY_POLL_INTERVAL = timeInSecond * 5;
 const INITIAL_RECOVERY_RUNS = 100;
 const RECOVERY_POLL_INTERVAL = timeInMinute;
@@ -213,7 +215,7 @@ class NodePathAuthorities {
             let obj = this.authorities.get(nodeId);
             if (!obj) {
                 // Might as well use this to add the node, if we don't know about it yet.
-                ingestNewNodeIds([nodeId], []);
+                void ingestNewNodeIds([nodeId], []);
                 return;
             }
             obj.authorityPaths = await PathController.nodes[nodeId].getAuthorityPaths();
@@ -225,7 +227,7 @@ class NodePathAuthorities {
         let firstPromise = new PromiseObj<unknown>();
         logErrors(firstPromise.promise);
 
-        const ingestNewNodeIds = (newNodeIds: string[], removedNodeIds: string[]) => {
+        const ingestNewNodeIds = runInSerial(async (newNodeIds: string[], removedNodeIds: string[]) => {
             for (let nodeId of removedNodeIds) {
                 this.authorities.delete(nodeId);
             }
@@ -249,6 +251,18 @@ class NodePathAuthorities {
                     return;
                 }
 
+                // Do an initial test by just opening a TCP connection. This will help filter out a lot of dead nodes immediately. Which, in turn, avoids having to create our initial connection signature, which saves a lot of time.
+                if (isNode()) {
+                    let nodeIdObj = getNodeIdLocation(nodeId);
+                    if (!nodeIdObj) {
+                        console.error(`Bad nodeId ${nodeId}`);
+                        return;
+                    }
+                    let isListening = await testTCPIsListening(nodeIdObj.address, nodeIdObj.port);
+                    if (!isListening) return;
+                }
+
+                console.log(blue(`Checking for status of node ${nodeId}`));
                 let time = Date.now();
                 let createTime = await timeoutToUndefinedSilent(POLL_RATE, PathController.nodes[nodeId].getCreateTime());
                 if (createTime === undefined) {
@@ -258,6 +272,7 @@ class NodePathAuthorities {
                             console.log(yellow(`Node didn't respond to getCreateTime`), { nodeId });
                         }
                     }
+                    console.log(yellow(`Node ${nodeId} is not available`));
                     this.previouslyNotAvailableNodes.add(nodeId);
                     return;
                 }
@@ -311,7 +326,7 @@ class NodePathAuthorities {
             if (isCurrentFirst) {
                 firstPromise.resolve(promise);
             }
-        };
+        });
 
         let time = Date.now();
         watchDeltaNodeIds(obj => ingestNewNodeIds(obj.newNodeIds, obj.removedNodeIds));
@@ -340,7 +355,7 @@ class NodePathAuthorities {
                 nonExistentNodes.push(node);
             }
             // Pretend all dead nodes are new
-            ingestNewNodeIds(nonExistentNodes, []);
+            await ingestNewNodeIds(nonExistentNodes, []);
         });
 
         let readyCount = 0;

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -23,6 +23,8 @@ import { TypedConfigEditor } from "../../library-components/TypedConfigEditor";
 import { managementPageURL } from "../../diagnostics/managementPages";
 import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
 import { getScreenName } from "../machineApplyMainCode";
+import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
+import { decodeNodeId } from "../../-a-auth/certs";
 
 
 
@@ -338,12 +340,12 @@ export class ServiceDetailPage extends qreact.Component {
                             <div
                                 className={css.hbox(12)}
                             >
-                                <div className={css.vbox(5)}>
+                                <div className={css.hbox(5)}>
                                     <div>
                                         {screenName}
                                     </div>
                                     <div>
-                                        {machineId} ({machineInfo.info["getExternalIP"]})
+                                        {serviceInfo?.nodeId || machineId} ({machineInfo.info["getExternalIP"]})
                                     </div>
                                 </div>
                                 {isDisabled && (

package/src/deployManager/machineSchema.ts

@@ -59,6 +59,8 @@ export type MachineInfo = {
         // Only times launched for the current applyNodeId, but... still very useful.
         totalTimesLaunched: number;
         // Might take a while to set (15 minutes or more). It's better to look in the nodes list and find the one that seems to match (maybe looking at the startup path to find it?)
+        // TODO: Make this work correctly when we deploy multiple instances (the node needs to know which index it is so it can write to a unique file).
+        //  - We could probably set environment variables in the screen?
         nodeId: string;
     }>;
 };

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -1,64 +1,20 @@
 /*
 todonext
 
-OKAY! CORE CONCEPTS
-    - Index needs to be fast searched, so it's uncompressed form has to be small
-        - Nesting compression is fine, as long as it's small enough that each search decompresses very little
-    - The enemy is storing every single position, we get around this by:
-        - In-exact matches, only storing block indexes.
-            - Requires full search to verify, but... in practice has low false hit rate
-        - Input key (unit) is only inexact matched.
-            - Input query almost always has more than 1 byte of uniqueness. This means if even part is only 50% accurate, we can still get very high hit rates (8 bytes means 255/256 hit rate!)
 
 
-todonext
-
-
-
-IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs should ever break. The code should be waiting until everything's fully flushed before it allows the shutdown handler to finish running. If we see any more errors, we need to investigate them. 
-
-
-
-
-//todonext
-
-
-
-3) "guessed node id" is never working? At least, on the machine's page it is ALWAYS empty, even though it should be almost always detected...
-
-4) Verify it works again, when we have multiple servers
-
-Invalid addition, you tried to add 1 + Infinity, operands must be valid numbers
-
-
-
-0) Add LZ4 compression to socket-function by default
-    - Setup local tester for it to start
-        - with non-synced endpoint, just surely socket-function
-        - purely nodejs, just talking to itself
-    - Allow setting "compress" to "none" or "lz4" or "zip" or "zip0" or "zip3", etc, for levels.
-        - default is "lz4"
-    - REQUIRES feature checking the remote, to make sure it is new enough to accept this.
-        - A generic thing which gets the version is probably fine.
-    - When decoding, I think we should just check the first byte. It should be a magic number which tells us which compression it's using. 
-    - LZ4 compression is fast enough that this should cause basically no overhead, and in many cases greatly reduce the bandwidth (which will increase the speed).
-    - We're gonna have to investigate how we're sending buffers anyway. I think this should be easy, but we 
-0.1) Verify the size difference with some local testing
-    - ALSO, verify the processing overhead is acceptable.
-1) Deploy, which SHOULD be backwards compatible with everything?
+2) Publish socket-function and update to use the latest
+1) Deploy everything and make sure the servers run
 
 
 -1) Change PathValueSerialize to use LZ4
-    - I think we already have a compression flag here, so it should be easy enough to just support it to change it to support LZ4. 
-    - I think we want to disable the server, and then when we launch the server locally, it should have to read from disk. 
--2) It would probably be valuable to test the synchronization speed between two different servers. It seems like when the remote is running and we start a local server, it really takes a long time for it to get ready. As in minutes. 
-    - This might have already been fixed by us improving the compression speed to not use gzip, maybe, but we should test it again, and if not, we should see what's slow
+    - I think we already have a compression flag here, so it should be easy enough to just change it to support and use LZ4 by default
+-3) Send progress/timing when sending very large messages?
+    - Maybe related to the remote => local synchronization. Maybe we're doing multi- many calls, in which case we can't detect it. But if we're doing just one call... Then the socket function system should realize. 
+        - And even if it's the synchronization code which is doing many calls, then it should probably log progress and timing (on both ends)
+-2) Fix synchronization speed from remote to local.
+    - Last time it seemed very slow
         - We might need to jump into the server and look at its profiles there by forcing them to omit, which isn't too hard, but it is kind of annoying. 
-
--3) During server startup (when there is a remote server), we spend a lot of time in identityHook.
-    - BUT... Our check to see if the async call that we're doing underneath is taking more than 200 milliseconds, is never firing. Even though apparently plenty of calls are taking over a second in total, so what why is it slow? Are some of the synchronous functions we're calling slow? 
-    - Basically, there's nowhere that could be slow, and we're timing it, and it times as being fast. However, in the profiles, it shows up as being slow. 
-        - It could also be an asynchronous issue, so some other timing is getting alias to it. That is feasible...
 */
 
 
@@ -73,6 +29,9 @@ Invalid addition, you tried to add 1 + Infinity, operands must be valid numbers
 //      - I mean, we have to implement it without caching anyways to start, so we can just do that, and then cache it later if we see the need...
 //          - WELL, we need SOME kind of caching... Maybe... we DO use a strict time range, and then... if we've scanned a pending file in the past, we can cache that, because we know we've got all of it's values. Hmm...
 //  - Use lifecycles to debug rejections. To a point, until it is likely we are out of sync, then we should write the sync verification code.
+//      - Maybe it isn't necessarily rejections, but definitely if we're running stuff locally and remotely it breaks. Could it be the case that we can't run the path value server both locally and remotely? And if so, we really should prevent this in some way. 
+//      - Annoyingly enough, we're actually going to have to add a case where we can scan the local logs as well as the remote logs to debug running it locally...
+//          - ALTHOUGH, It does fail just if we have the local running server, But we're on the remote site. So we might be able to just debug it from that life cycle, from the remote life cycles. 
 
 
 // todonext;

package/src/storage/LZ4.ts

@@ -1,32 +1,2 @@
-// NOTE: Even if we wanted to use the production version, we couldn't because it's not compatible with the client-side code, because they decided to do a file read to load in their WebAssembly. 
-import lz4_stream from "../misc/lz4_wasm_nodejs";
-import { measureFnc } from "socket-function/src/profiling/measure";
-export class LZ4 {
-    @measureFnc
-    static compress(data: Buffer): Buffer {
-        return this.compressUntracked(data);
-    }
-    static compressUntracked(data: Buffer): Buffer {
-        try {
-            return Buffer.from(lz4_stream.compress(data));
-        } catch (e) {
-            // Rethrow non errors as properly wrapped errors
-            if (!(e && e instanceof Error)) {
-                throw new Error(`Error compressing LZ4: ${e}`);
-            }
-            throw e;
-        }
-    }
-    @measureFnc
-    static decompress(data: Buffer): Buffer {
-        try {
-            return Buffer.from(lz4_stream.decompress(data));
-        } catch (e) {
-            // Rethrow non errors as properly wrapped errors
-            if (!(e && e instanceof Error)) {
-                throw new Error(`Error decompressing LZ4: ${e}`);
-            }
-            throw e;
-        }
-    }
-}
+import { LZ4 } from "socket-function/src/lz4/LZ4";
+export { LZ4 };

package/test.ts

@@ -6,12 +6,39 @@ import "./inject";
 import { Querysub } from "./src/4-querysub/QuerysubController";
 import { getErrorLogs, getLoggers2Async } from "./src/diagnostics/logs/diskLogger";
 import { watchAllValues } from "./src/diagnostics/logs/errorNotifications2/logWatcher";
+import { SocketFunction } from "socket-function/SocketFunction";
+import fs from "fs";
+import { getControllerNodeId, getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
+import { delay } from "socket-function/src/batching";
+import { timeInSecond } from "socket-function/src/misc";
 
+const path = "D:/repos/qs-cyoa/database-storage/disklogs/1757030400000-1757116800000.log";
+class TestControllerBase {
+    async test() {
+        return await fs.promises.readFile(path, "utf8");
+    }
+}
+
+const TestController = SocketFunction.register("TestController-019ca5fa-52ef-73bf-b918-4d9cde51b618", new TestControllerBase(), () => ({
+    test: {
+        compress: true,
+    },
+}));
 async function main() {
     await Querysub.hostService("testwatcher");
-    let errorLogs = await getErrorLogs();
-    for await (let error of watchAllValues(errorLogs)) {
-        process.stdout.write(JSON.stringify(error) + "\n");
+
+    while (true) {
+        try {
+            let otherNode = await getControllerNodeId(TestController);
+            if (!otherNode) throw new Error("No other node found");
+            let result = await TestController.nodes[otherNode].test();
+            let correct = await fs.promises.readFile(path, "utf8");
+            if (result !== correct) throw new Error("Result does not match correct");
+            console.log(`Received ${result.length} bytes from ${otherNode}`);
+        } catch (e: any) {
+            console.error(e);
+        }
+        await delay(timeInSecond * 1);
     }
 }