querysub 0.392.0 → 0.393.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.392.0",
+  "version": "0.393.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -57,7 +57,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "querysub": "^0.357.0",
-    "socket-function": "^1.1.4",
+    "socket-function": "^1.1.5",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -17,6 +17,8 @@ setFlag(require, "cbor-x", "allowclient", true);
 
 import * as pako from "pako";
 import { delay } from "socket-function/src/batching";
+import { LZ4 } from "../storage/LZ4";
+import { unblockLoop } from "socket-function/src/batching";
 setFlag(require, "pako", "allowclient", true);
 
 const DEFAULT_BUFFER_SIZE = 1024 * 16;
@@ -32,6 +34,7 @@ const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
 
 const SERIALIZE_LOCK_COUNT = true;
 
+
 /*
     Memory Layout
         DataSettings
@@ -67,7 +70,7 @@ interface Reader {
 
 // JSON.stringify, length prefixed
 interface DataSettings {
-    compression?: "gzip";
+    compression?: "gzip" | "lz4";
     valueCount: number;
     noLocks?: boolean;
     stripSource?: boolean;
@@ -390,7 +393,7 @@ class PathValueSerializer {
         let settings: DataSettings = {
             valueCount: values.length,
             noLocks: config?.noLocks,
-            compression: config?.compress ? "gzip" : undefined,
+            compression: config?.compress ? "lz4" : undefined,
             stripSource: config?.stripSource,
             version,
         };
@@ -510,7 +513,20 @@ class PathValueSerializer {
         stringBuffers.reverse();
         outputBuffers.push(...stringBuffers);
 
-        if (settings.compression === "gzip") {
+        if (settings.compression === "lz4") {
+            let compressedBuffers = await unblockLoop(outputBuffers.slice(1), x => LZ4.compress(x));
+            let compressedOutputBuffers = [outputBuffers[0], ...compressedBuffers];
+
+            // If the compress factor is less than a threshold, use the uncompressed buffers
+            let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
+            let compressedSize = compressedOutputBuffers.reduce((total, x) => total + x.length, 0);
+            if (compressedSize / uncompressedSize < MIN_COMPRESS_FACTOR) {
+                outputBuffers = compressedOutputBuffers;
+            } else {
+                settings.compression = undefined;
+                outputBuffers[0] = Buffer.from(JSON.stringify(settings));
+            }
+        } else if (settings.compression === "gzip") {
             // NOTE: Due to how the LZ77 window works merging buffers probably won't reduce the size by that much.
             let compressedBuffers = await Promise.all(outputBuffers.slice(1).map(x => Zip.gzip(x, 1)));
             let compressedOutputBuffers = [outputBuffers[0], ...compressedBuffers];
@@ -574,7 +590,9 @@ class PathValueSerializer {
 
         let settings = JSON.parse(buffers.shift()!.toString("utf8")) as DataSettings;
         let version = settings.version || 0;
-        if (settings.compression === "gzip") {
+        if (settings.compression === "lz4") {
+            buffers = await unblockLoop(buffers, x => LZ4.decompress(x));
+        } else if (settings.compression === "gzip") {
             buffers = await Zip.gunzipBatch(buffers);
         }
 

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -23,6 +23,7 @@ import { getBufferFraction, getBufferInt, getShortNumber } from "../bits";
 import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { testTCPIsListening } from "socket-function/src/networking";
+import { getMachineId } from "../-a-auth/certs";
 
 export const LOCAL_DOMAIN = "LOCAL";
 export const LOCAL_DOMAIN_PATH = getPathStr1(LOCAL_DOMAIN);
@@ -87,6 +88,8 @@ class NodePathAuthorities {
 
     private selfAuthorities: AuthorityPath[] = [];
 
+    private outdatedNodeIds = new Set<string>();
+
     public debug_getAuthority(nodeId: string) {
         return this.authorities.get(nodeId);
     }
@@ -251,6 +254,13 @@ class NodePathAuthorities {
                     return;
                 }
 
+                let nodeMachineId = getMachineId(nodeId);
+                let onSameMachine = Array.from(this.authorities.keys()).filter(x => getMachineId(x) === nodeMachineId && x !== nodeId);
+                if (this.outdatedNodeIds.has(nodeId) && onSameMachine.length > 0) {
+                    // It's just an old thread that's dead, and once threads die, they don't come back. This should be fairly safe, as we confirm that both a new thread is running on the same port, and that the old thread doesn't respond. 
+                    return;
+                }
+
                 // Do an initial test by just opening a TCP connection. This will help filter out a lot of dead nodes immediately. Which, in turn, avoids having to create our initial connection signature, which saves a lot of time.
                 if (isNode()) {
                     let nodeIdObj = getNodeIdLocation(nodeId);
@@ -266,6 +276,15 @@ class NodePathAuthorities {
                 let time = Date.now();
                 let createTime = await timeoutToUndefinedSilent(POLL_RATE, PathController.nodes[nodeId].getCreateTime());
                 if (createTime === undefined) {
+                    if (!isNodeIdLocal(nodeId)) {
+                        let aliveNodes = await Promise.all(onSameMachine.map(x => errorToUndefinedSilent(PathController.nodes[x].isReadReady())));
+                        if (aliveNodes.some(x => x)) {
+                            console.warn(`Node is on the same machine as port as another alive node. AND, it is not responding. Assuming it is just an older version of a dead node`, { nodeId, aliveNodes });
+                            this.outdatedNodeIds.add(nodeId);
+                            return;
+                        }
+                    }
+
                     // Don't log for 127-0-0-1, as it usually fails, and is mostly a development optimization
                     if (!nodeId.includes("127-0-0-1")) {
                         if (!this.previouslyNotAvailableNodes.has(nodeId)) {

package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts

@@ -3,7 +3,7 @@ import { nestArchives } from "../../../-a-archives/archives";
 import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
 import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
 import { Querysub } from "../../../4-querysub/QuerysubController";
-import { getDomain } from "../../../config";
+import { getDomain, isPublic } from "../../../config";
 import { MachineInfo } from "../../../deployManager/machineSchema";
 import { createMatchesPattern } from "../IndexedLogs/bufferSearchFindMatcher";
 import { LogDatum, getErrorLogs } from "../diskLogger";
@@ -482,7 +482,7 @@ const ErrorNotificationServiceBase = SocketFunction.register(
 
 class ErrorNotificationData {
     public async getData() {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }
@@ -490,7 +490,7 @@ class ErrorNotificationData {
     }
 
     public async getUnmatchedErrorsLimited() {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }
@@ -536,7 +536,7 @@ class ErrorNotificationData {
     }
 
     private static ensureWatchingErrorsHTTP = lazy(async () => {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             ErrorNotificationData.ensureWatchingErrorsHTTP.reset();
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
@@ -572,7 +572,7 @@ class ErrorNotificationData {
     }
 
     public async getSuppressionEntries() {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }
@@ -580,7 +580,7 @@ class ErrorNotificationData {
     }
 
     public async setSuppressionEntry(entry: SuppressionEntry) {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }
@@ -588,7 +588,7 @@ class ErrorNotificationData {
     }
 
     public async deleteSuppressionEntry(id: string) {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }
@@ -600,7 +600,7 @@ class ErrorNotificationData {
     }
 
     public async updateSuppressionNotes(id: string, notes: string | undefined) {
-        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase, !isPublic());
         if (!controllerNodeId) {
             throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
         }

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -1,20 +1,14 @@
 /*
 todonext
 
+-1) Deploy everything again!
 
+-2) Fix synchronization speed from remote to local.
+    - I think we just have a lot of data, which our code will detect and log now.
+        - And we'll see how much data we are even sending
 
-2) Publish socket-function and update to use the latest
-1) Deploy everything and make sure the servers run
-
+Write Monthly Summary
 
--1) Change PathValueSerialize to use LZ4
-    - I think we already have a compression flag here, so it should be easy enough to just change it to support and use LZ4 by default
--3) Send progress/timing when sending very large messages?
-    - Maybe related to the remote => local synchronization. Maybe we're doing multi- many calls, in which case we can't detect it. But if we're doing just one call... Then the socket function system should realize. 
-        - And even if it's the synchronization code which is doing many calls, then it should probably log progress and timing (on both ends)
--2) Fix synchronization speed from remote to local.
-    - Last time it seemed very slow
-        - We might need to jump into the server and look at its profiles there by forcing them to omit, which isn't too hard, but it is kind of annoying. 
 */
 
 

package/test.ts

@@ -10,31 +10,97 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import fs from "fs";
 import { getControllerNodeId, getControllerNodeIdList } from "./src/-g-core-values/NodeCapabilities";
 import { delay } from "socket-function/src/batching";
-import { timeInSecond } from "socket-function/src/misc";
+import { list, timeInSecond } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { LZ4 } from "socket-function/src/lz4/LZ4";
+
+const paths = [
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756166400000-1756252800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-0-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756684800000-1756771200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756684800000-1756771200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/bootstrapper-0-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/bootstrapper-0-dply-1756339200000-1756425600000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756339200000-1756425600000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756339200000-1756425600000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/bootstrapper-0-dply-1756684800000-1756771200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/function-0-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/bootstrapper-0-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/bootstrapper-0-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/function-0-dply-1756684800000-1756771200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/function-0-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-0-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-1-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-0-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-1-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756166400000-1756252800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756166400000-1756252800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/function-0-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-0-dply-1756080000000-1756166400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/gc-0-dply-1756425600000-1756512000000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756339200000-1756425600000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/gc-0-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/server-1-dply-1756080000000-1756166400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/gc-0-dply-1756339200000-1756425600000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756598400000-1756684800000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756684800000-1756771200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/cyoa-1-dply-1756252800000-1756339200000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756512000000-1756598400000.log",
+    "D:/repos/qs-cyoa/database-storage/disklogs/join-0-dply-1756080000000-1756166400000.log",
+];
+
+const FACTOR = 1;
+
+
+let getData = lazy(async () => {
+    let buffers: Buffer[] = [];
+    for (let path of paths) {
+        let data = await fs.promises.readFile(path, "utf8");
+        buffers.push(Buffer.from(data));
+    }
+    return list(FACTOR).map(i => buffers.map(b => Buffer.from(b))).flat();
+});
 
-const path = "D:/repos/qs-cyoa/database-storage/disklogs/1757030400000-1757116800000.log";
 class TestControllerBase {
     async test() {
-        return await fs.promises.readFile(path, "utf8");
+        return (await getData()).slice();
     }
 }
 
 const TestController = SocketFunction.register("TestController-019ca5fa-52ef-73bf-b918-4d9cde51b618", new TestControllerBase(), () => ({
     test: {
-        compress: true,
+        //compress: false
     },
 }));
+function compareBuffers(result: Buffer[], correct: Buffer[]) {
+    if (result.length !== correct.length) throw new Error(`Result does not match correct: ${result.length} !== ${correct.length}`);
+    for (let i = 0; i < result.length; i++) {
+        if (!result[i].equals(correct[i])) throw new Error(`Result does not match correct: ${result[i].length} !== ${correct[i].length}, at index ${i}`);
+    }
+}
 async function main() {
     await Querysub.hostService("testwatcher");
+    //SocketFunction.logMessages = true;
 
     while (true) {
         try {
             let otherNode = await getControllerNodeId(TestController);
             if (!otherNode) throw new Error("No other node found");
-            let result = await TestController.nodes[otherNode].test();
-            let correct = await fs.promises.readFile(path, "utf8");
-            if (result !== correct) throw new Error("Result does not match correct");
-            console.log(`Received ${result.length} bytes from ${otherNode}`);
+            let result = (await TestController.nodes[otherNode].test());
+            let correct = (await getData());
+            compareBuffers(result, correct);
+            console.log(`Received ${formatNumber(result.reduce((a, b) => a + b.length, 0))} bytes from ${otherNode}`);
         } catch (e: any) {
             console.error(e);
         }