querysub 0.144.0 → 0.146.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.144.0",
+  "version": "0.146.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "preact": "^10.11.3",
-    "socket-function": "^0.80.0",
+    "socket-function": "^0.82.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.6.3",
     "yaml": "^2.5.0",

package/src/0-path-value-core/PathValueCommitter.ts

@@ -11,7 +11,7 @@ import { errorToUndefined, logErrors, timeoutToUndefined } from "../errors";
 import { getPathFromStr } from "../path";
 import { nodePathAuthority, pathValueAuthority2 } from "./NodePathAuthorities";
 import { PathValueController } from "./PathValueController";
-import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath, compareTime, epochTime, isOurPrediction } from "./pathValueCore";
+import { PathValue, MAX_ACCEPTED_CHANGE_AGE, lockWatcher, authorityStorage, writeValidStorage, WriteState, lockToCallback, pathWatcher, MAX_CHANGE_AGE, debugPathValue, debugPathValuePath, compareTime, epochTime, isOurPrediction, timeHash } from "./pathValueCore";
 import debugbreak from "debugbreak";
 import { red } from "socket-function/src/formatting/logColors";
 import { registerDynamicResource } from "../diagnostics/trackResources";
@@ -278,6 +278,23 @@ class PathValueCommitter {
             //  missing values (unless we are the authority). Otherwise bad rejected values can stick around,
             //  because they are newer, and our new source doesn't know we have them.
             measureBlock(function resetOnInitialTrigger() {
+                let receivedValues = new Map<string, Set<string>>();
+                for (let batch of batched) {
+                    for (let pathValue of batch.pathValues) {
+                        let path = pathValue.path;
+                        let hash = timeHash(pathValue.time);
+                        let paths = receivedValues.get(path);
+                        if (!paths) {
+                            paths = new Set();
+                            receivedValues.set(path, paths);
+                        }
+                        paths.add(hash);
+                    }
+                }
+
+                // TODO: Ugh... rejections on initialSync might the wrong solution. I think we have to just remove the values,
+                //  even though this makes debugging harder, in order to prevent the rejections from sticking around.
+
                 for (let batch of batched) {
                     if (!batch.initialTrigger) continue;
                     for (let pathValue of batch.pathValues) {
@@ -292,12 +309,24 @@ class PathValueCommitter {
                             // Only reset values after us, otherwise we might clear valid history that
                             //  we JUST received.
                             if (compareTime(value.time, pathValue.time) <= 0) continue;
+
+                            // If we just received it, accept it.
+                            //  - Otherwise we might reject a valid older value, and then when the newer value is rejected
+                            //      the server won't tell us about the older value again, and so we will think no values
+                            //      are accepted, when the older value IS accepted.
+                            // BUG: In theory couldn't we receive an outdated value, then reconnect, then receive the correct
+                            //  value, and have them all batched together? In which case, we shouldn't accept the value.
+                            //  I'm not really sure how to fix this though. We kind of need to just handle adding the values
+                            //  and the rejections in order, instead of batching them, but that is much slower, and the race
+                            //  condition will likely be extremely rare (and only happen on disconnections anyways).
+                            if (receivedValues.get(pathValue.path)?.has(timeHash(pathValue.time))) continue;
+
                             specialInitialParentCausedRejections.push({
                                 path: value.path,
                                 time: value.time,
                                 isValid: false,
                                 isTransparent: !!value.isTransparent,
-                                reason: "future missing (initial trigger)",
+                                reason: "future missing on resync (initial trigger)",
                             });
                         }
                     }
@@ -431,6 +460,13 @@ class PathValueCommitter {
 
         let now = Date.now();
 
+        // NOTE: specialInitialParentCausedRejections is a bit dangerous, because it can reject golden values.
+        //  This means we have to ingest it immediately, otherwise authorityStorage might gc path values
+        //  (that are before a golden value), when rejections might remove that golden value.
+        if (specialInitialParentCausedRejections) {
+            this.ingestValidStates(specialInitialParentCausedRejections);
+        }
+
         authorityStorage.ingestValues(pathValues, parentsSynced, parentSyncedSources);
 
         // NOTE: This doesn't seem to be where we lag, so we aren't tracking it anymore.
@@ -440,7 +476,7 @@ class PathValueCommitter {
         }
 
         // NOTE: Also triggers rejections of watched paths
-        this.ingestValidStates(specialInitialParentCausedRejections ?? [], parentsSynced, undefined, pathValues, initialTriggers);
+        this.ingestValidStates([], parentsSynced, undefined, pathValues, initialTriggers);
     }
 
     // NOTE: Technically, because we trigger all remotes any time a value changes, even

package/src/0-path-value-core/pathValueCore.ts

@@ -945,9 +945,10 @@ class AuthorityPathValueStorage {
         //  some values in memory...
 
         let goldenTime = now - MAX_CHANGE_AGE;
-        // NOTE: Values with no locks are implicitly golden, as they can't be rejected.
-        let firstGoldenIndex = values.findIndex(x => x.time.time < goldenTime || x.lockCount === 0);
+        // NOTE: Values with no locks are implicitly golden, as they can't be rejected
+        let firstGoldenIndex = values.findIndex(x => x.time.time < goldenTime && x.valid || x.lockCount === 0);
         if (firstGoldenIndex < 0) return;
+
         // Special case, everything is golden, and the latest value is undefined, and is behind the GC
         //  point, then delete everything, so constantly changing keys doesn't result in us leaking memory forever.
         if (
@@ -987,9 +988,8 @@ class AuthorityPathValueStorage {
             }
         } else {
             // Delete everything that is golden, except for the most recent VALID value.
-            let validGoldenIndex = values.findIndex(x => x.valid && x.time.time < goldenTime);
-            if (validGoldenIndex >= 0 && values.length > validGoldenIndex) {
-                for (let i = values.length - 1; i > validGoldenIndex; i--) {
+            if (firstGoldenIndex >= 0 && values.length > firstGoldenIndex) {
+                for (let i = values.length - 1; i > firstGoldenIndex; i--) {
                     let gced = values.pop();
                     if (!isCoreQuiet && gced) {
                         console.log(`GCing ${debugPathValuePath(gced)}`);

package/src/diagnostics/ActionsHistory.ts

@@ -33,24 +33,30 @@ export class ActionsHistory {
     }
     public static OnRead(values: PathValue[]) {
         if (ClientWatcher.DEBUG_READS && values.length > 0) {
-            let now = Date.now();
-            let oldest = Math.min(...values.map(x => x.time.time).filter(x => x), now);
-            group(ClientWatcher.DEBUG_READS_EXPANDED)(`(${now - epoch}ms, ${now}) Read count ${values.length}, ${(Date.now() - oldest).toFixed()}ms old`);
-            for (let value of values) {
-                console.log(blue(debugPathValuePath(value)), "===", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+            values = values.filter(x => !x.path.startsWith(LOCAL_DOMAIN_PATH));
+            if (values.length > 0) {
+                let now = Date.now();
+                let oldest = Math.min(...values.map(x => x.time.time).filter(x => x), now);
+                group(ClientWatcher.DEBUG_READS_EXPANDED)(`(${now - epoch}ms, ${now}) Read count ${values.length}, ${(Date.now() - oldest).toFixed()}ms old`);
+                for (let value of values) {
+                    console.log(blue(debugPathValuePath(value)), "===", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+                }
+                console.groupEnd();
             }
-            console.groupEnd();
         }
         if (!ActionsHistory.LOG_ACTION_HISTORY) return;
     }
     public static OnWrite(values: PathValue[]) {
         if (ClientWatcher.DEBUG_WRITES) {
-            let now = Date.now();
-            group(ClientWatcher.DEBUG_WRITES_EXPANDED)(green(`(${now - epoch}ms, ${now}) Committing writes ${values.length}`));
-            for (let value of values) {
-                console.log(green(debugPathValuePath(value)), "=", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+            values = values.filter(x => !x.path.startsWith(LOCAL_DOMAIN_PATH));
+            if (values.length > 0) {
+                let now = Date.now();
+                group(ClientWatcher.DEBUG_WRITES_EXPANDED)(green(`(${now - epoch}ms, ${now}) Committing writes ${values.length}`));
+                for (let value of values) {
+                    console.log(green(debugPathValuePath(value)), "=", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+                }
+                console.groupEnd();
             }
-            console.groupEnd();
         }
         if (!ActionsHistory.LOG_ACTION_HISTORY) return;
     }