querysub 0.487.0 → 0.489.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.487.0",
+  "version": "0.489.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathValueController.ts

@@ -21,6 +21,7 @@ import { Time } from "./pathValueCore";
 import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
 import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
 import { delay } from "socket-function/src/batching";
+import { isDefined } from "../misc";
 export { pathValueCommitter };
 
 let pathValueSendCount = 0;
@@ -103,12 +104,14 @@ export class PathValueControllerBase {
         pathValues: PathValue[];
         initialTriggers?: { values: Set<string>; parentPaths: Set<string> },
         reason: string;
+        // Locks are normally stripped because watchers don't need them, but a receiving authority needs the locks to compute the valid state itself — without them it can only adopt our valid state and can never recompute.
+        keepLocks?: boolean;
     }) {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
         let buffers = await pathValueSerializer.serialize(changes, {
-            noLocks: true,
+            noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
         });
         this.logSendValues({ nodeId, pathValues: changes, initialTriggers, reason: config.reason });
@@ -182,6 +185,9 @@ export class PathValueControllerBase {
             }
         }
 
+        // Note which values are genuinely new to us BEFORE ingesting. We only re-share new values with the other authorities below; if we already had a value we already shared it when we first received it, so re-sharing is redundant and can clobber a newer value with an older copy.
+        let valuesNewToUs = config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [];
+
         try {
             let initialTriggers = config.initialTriggers || { values: new Set(), parentPaths: new Set() };
             await pathValueCommitter.ingestRemoteValuesAndValidStates({
@@ -194,9 +200,10 @@ export class PathValueControllerBase {
         }
 
 
-        if (config.initialCreation) {
-            // NOTE: This will mean that all of the authorities for a value will write it to disk. So it will be written to disk multiple times. This should be fine, and it might be a good idea for redundancy, etc. And eventually, they will get compressed to one value, so it's not inefficient.
-            await PathValueControllerBase.authorityShareValues({ pathValues: values });
+        if (config.initialCreation && valuesNewToUs.length > 0) {
+            // Always shared the latest. If we don't have it, maybe something with ingestion delayed, we should still share it so we don't lose the value. If we haven't even ingested it, then it is our latest version of the value.
+            valuesNewToUs = valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value);
+            await PathValueControllerBase.authorityShareValues({ pathValues: valuesNewToUs });
         }
     }
 
@@ -211,6 +218,7 @@ export class PathValueControllerBase {
                     nodeId: otherAuthority,
                     pathValues: values,
                     reason: "authorityShareValues",
+                    keepLocks: true,
                 }), () => new Error(`Timeout forwarding shared values (${values.length} values) to authority ${otherAuthority}`));
             } catch (error: any) {
                 console.error(error.message, { otherAuthority, count: values.length });

package/src/0-path-value-core/ValidStateComputer.ts

@@ -92,11 +92,7 @@ class ValidStateComputer {
             return true;
         });
 
-        // For any value we're the authority on that we already have, swap in our own stored copy. The data
-        //  is immutable per (path, time) — only the valid state can differ, and we own that — so we must never
-        //  adopt a sibling authority's valid state. Remapping (rather than only skipping the re-ingest) keeps
-        //  everything downstream (valuesChanged, dependenciesChanged) on a single object instead of a mix of
-        //  our copy and the remote's.
+        // For any value we're the authority on that we already have, swap in our own stored copy. The data is immutable per (path, time) — only the valid state can differ, and we own that — so we must never adopt a sibling authority's valid state. Remapping (rather than only skipping the re-ingest) keeps everything downstream (valuesChanged, dependenciesChanged) on a single object instead of a mix of our copy and the remote's.
         pathValues = pathValues.map(value => {
             if (!PathRouter.isSelfAuthority(value.path)) return value;
             return authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value;
@@ -125,7 +121,9 @@ class ValidStateComputer {
                 dependenciesChanged.add(watcher);
             }
 
-            if (!value.valid) {
+            // We don't compute the valid state for values we don't own, but we still forward them to our watchers, and that must include accept transitions, not only rejects — otherwise a watcher we already forwarded a reject to stays stuck once the authority accepts. Ingest happens further below, so getValueExactMaybeRejected still returns our previous value here; only trigger when the incoming valid differs from what we currently hold, so we forward real changes without re-triggering on unchanged re-sends.
+            let prevValidState = authorityStorage.getValueExactMaybeRejected(value.path, value.time)?.valid;
+            if (prevValidState !== value.valid) {
                 initialTriggers.initialTriggerNonHistoryWatchers.add(value.path);
             }
         }
@@ -232,10 +230,8 @@ class ValidStateComputer {
             for (let validStateChange of validStateChanged) {
                 valuesChanged.add(validStateChange);
 
-                // NOTE: This approach should be faster than doing a full initial sync, but we'll see if it breaks anything.
-                if (!validStateChange.valid) {
-                    initialTriggers.initialTriggerNonHistoryWatchers.add(validStateChange.path);
-                }
+                // Push the latest value to this path's (possibly cross-node) watchers on ANY valid-state change, not just rejections — external watchers are only re-notified via initialTriggerNonHistoryWatchers; valuesChanged alone propagates between co-authorities and never re-notifies a watcher that already holds a stale verdict, so without notifying on accepts a reject→accept recovery is computed correctly on the authority but the watcher stays stuck on the earlier rejected value.
+                initialTriggers.initialTriggerNonHistoryWatchers.add(validStateChange.path);
 
                 // // NOTE: As of right now, I think the only reason to force an initial trigger in this case is so that we can bust all the values through the query sub http server to the clients. Because if we just tell the HTTP server that one value has been rejected, it can go tell its clients, but that won't give them the latest value. Which isn't great. We should really have something where we can tell them. HMM...
                 // // IMPORTANT! The consequence of this is that if someone predicts a value, sends it to us, it's not valid, we will see it's not valid, and we will tell all of the nodes that are watching that path about all of the values on that path. This is inefficient. However, rejected values should be quite rare, so it shouldn't be too inefficient.