querysub 0.426.0 → 0.428.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.426.0",
+  "version": "0.428.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathRouter.ts

@@ -10,6 +10,7 @@ import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
 import { sha256 } from "js-sha256";
 import { rangesOverlap, removeRange } from "../rangeMath";
+import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
 
 
 // Cases
@@ -57,6 +58,19 @@ function matchesPrefix(matcher: PrefixMatcher, path: string): boolean {
             return false;
         }
     }
+    if (getPathIndex(path, matcher.childKeyIndex) === undefined) return false;
+    return true;
+}
+// Checks if they're asking for the exact prefix, which would mean that the children would be the direct children of this prefix matcher. 
+function isPrefixParent(matcher: PrefixMatcher, path: string): boolean {
+    if (!path.startsWith(matcher.prefix)) return false;
+    let partsA = getPathFromStr(matcher.originalPrefix);
+    let partsB = getPathFromStr(path);
+    for (let i = 0; i < partsA.length; i++) {
+        // Empty string is wildcard
+        if (partsA[i] === "") continue;
+        if (partsB[i] !== partsA[i]) return false;
+    }
     return true;
 }
 /** Empty string path part becomes a wildcard */
@@ -535,11 +549,17 @@ export class PathRouter {
             range: { start: number; end: number };
         }[];
     } {
+        let parentRange = decodeParentFilter(path) || {
+            start: 0,
+            end: 1,
+        };
+        path = hack_stripPackedPath(path);
         let preferredNodeIds = new Set(config?.preferredNodeIds ?? []);
 
         // If a prefix is a parent of path, then it is the same as matching just the path directly
         // (If our prefix directly equals one of the other matches, then it's more complicated, As then, the child keys of path are what is hashed, and so all the children will have different routes, so we might match multiple nodes. The same thing if we're matching the remaining case, in which case it's a full path hash, so the child key matters, and again, different routes).
         //      - The different route case is how the FuntionRunner works, and without it large databases couldn't run functions. However, most applications won't directly use it.
+        // NOTE: The only own nodes flag is actually so we can access this before the topology finishes synchronizing. Because we want to be able to call get child read nodes from path value core for some really basic stuff. It also is what the caller wants, but that's not a good enough reason to add this check. The reason we have this check is because without it, the topology won't have finished synchronizing and startup won't work.
         let allSources = config?.onlyOwnNodes ? [{ nodeId: getOwnNodeId(), authoritySpec: authorityLookup.getOurSpec() }] : authorityLookup.getTopologySync();
         // Prefer our own node
         sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
@@ -549,14 +569,13 @@ export class PathRouter {
 
 
         // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
-        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y.prefix === path)).map(x => x.authoritySpec);
+        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => isPrefixParent(y, path))).map(x => x.authoritySpec);
         if (hasPrefix.length > 0) {
             shuffle(hasPrefix, Math.random());
             sort(hasPrefix, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
 
             let missingRanges: { start: number; end: number }[] = [{
-                start: 0,
-                end: 1,
+                ...parentRange
             }];
             let usedParts: {
                 nodeId: string;
@@ -585,7 +604,7 @@ export class PathRouter {
         let nestedMatches = allSources.filter(x => {
             // There's nested prefixes, so if we match any prefix explicitly, we can't just take one of the previous prefixes because that isn't how the hashing will work. 
             // - This happens if it's a direct match, but one of the shards is down, in which case we can't get a full match.
-            if (x.authoritySpec.prefixes.some(y => y.prefix === path)) return false;
+            if (x.authoritySpec.prefixes.some(y => isPrefixParent(y, path))) return false;
 
             // If our path, which we're going to read the children of, is the child of another path, then it means in that other path, the child key will be known to us constant, and so we're going to match exactly one authority. 
             return (
@@ -645,13 +664,11 @@ export class PathRouter {
         }
 
 
-
-
         if (!config?.onlyOwnNodes) {
             // TODO: We could maybe match a partial match. However, even that is suspect. The site being partially broken is almost worse than it being completely broken. We should just get ALL the shards running again...
 
             // NOTE: We *could* actually synchronize it even if it doesn't have a prefix shard as we can fall back to just the full path sharding. However, it becomes very complicated if we want a specific range, and then it becomes complicated if it then switches to prefix hashing (With the nodes that were using the full path hashing slowly going away). AND... key synchronization IS slow, so it's good to discourage it in general. 
-            console.error(`Want to sync a prefix which is not under an existing prefix, nor equal to a prefix. 1) The servers are down. 2) Don't access the .keys() 3) call addRoutingPrefixForDeploy to add a route/parent route explicitly (as is done in PathFunctionRunner.ts). Path: ${JSON.stringify(path)}`, { path, allSources });
+            console.error(`Want to sync a prefix which is not under an existing prefix, nor equal to a prefix. 1) The servers are down. 2) Don't access the .keys() 3) call addRoutingPrefixForDeploy to add a route/parent route explicitly (as is done in PathFunctionRunner.ts). Path: ${JSON.stringify(path)}`, { path, allSources: allSources.map(x => x.authoritySpec) });
         }
         return { nodes: [] };
     }

package/src/0-path-value-core/PathWatcher.ts

@@ -369,8 +369,10 @@ class PathWatcher {
 
         // initialTriggers.parentPaths => triggerPaths
         for (let parentPath of initialTriggers.parentPaths) {
-            let valuePaths = authorityStorage.getPathsFromParent(parentPath);
+            // NOTE: We have to trigger all of the values without the hacked path restriction. This should be safe, if a little bit inefficient, because we're still not going to send the values to watchers who aren't watching them. But we need to trigger all of it because it might be the case that we had synchronized 99% of the path and someone was watching 100% of the path and we just finished synchronizing the last 1%. So the parent path trigger that we received only has 1%, but we need to send everything. And it's too complicated to determine exactly what we need to send as in the intersection of all the watchers. So it's easier to just trigger all the value paths and what they're watching will naturally be sent. 
+            let valuePaths = authorityStorage.getPathsFromParent(hack_stripPackedPath(parentPath));
             for (let valuePath of valuePaths || []) {
+                if (!authorityStorage.isSynced(valuePath)) continue;
                 // IMPORTANT! Add all the values to the initial triggers so then later we not only trigger them but know their initial trigger so we can make sure that the is initial trigger logic for each value runs as well. 
                 initialTriggers.values.add(valuePath);
                 triggerPaths.add(valuePath);
@@ -378,7 +380,9 @@ class PathWatcher {
 
             let latestParentWatches = this.parentWatchers.get(hack_stripPackedPath(parentPath));
             if (!latestParentWatches) continue;
-            for (let { watchers } of latestParentWatches.values()) {
+            for (let [packedPath, { watchers }] of latestParentWatches) {
+                // ONLY trigger, if packedPath is fully synchronized
+                if (!authorityStorage.isParentSynced(packedPath)) continue;
                 for (let watcher of watchers) {
                     if (onlyTriggerNodeId && watcher !== onlyTriggerNodeId) continue;
                     let changes = changedPerCallbacks.get(watcher);
@@ -386,7 +390,7 @@ class PathWatcher {
                         changes = { values: new Set(), initialTriggers: { values: new Set(), parentPaths: new Set() } };
                         changedPerCallbacks.set(watcher, changes);
                     }
-                    changes.initialTriggers.parentPaths.add(parentPath);
+                    changes.initialTriggers.parentPaths.add(packedPath);
                 }
             }
         }

package/src/0-path-value-core/pathValueCore.ts

@@ -609,11 +609,12 @@ class AuthorityPathValueStorage {
         return false;
     }
     public isParentSynced(path: string) {
+        let originalPath = path;
         let range = decodeParentFilter(path);
         path = hack_stripPackedPath(path);
         if (PathRouter.isLocalPath(path)) return true;
 
-        let synced = this.parentsSynced.get(path);
+        let synced = this.parentsSynced.get(originalPath) || this.parentsSynced.get(path);
         if (synced === true) return true;
         // See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path. 
         if (synced && range) {
@@ -626,24 +627,11 @@ class AuthorityPathValueStorage {
 
         // Self authority check
         {
-            let nodes = PathRouter.getChildReadNodes(path, { onlyOwnNodes: true });
-            if (nodes.nodes.length > 0) {
-                if (nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
-                    // If it's not a partial 30 and we're self authority, we're always going to be self authority, so we can just cache it in the lookup. 
-                    this.parentsSynced.set(path, true);
-                    return true;
-                }
-                // Might be a partial authority
-                if (range) {
-                    let selfNodes = nodes.nodes.filter(x => isOwnNodeId(x.nodeId));
-                    let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
-                    for (let range of selfNodes) {
-                        removeRange(missingRanges, range.range);
-                    }
-                    if (missingRanges.length === 0) {
-                        return true;
-                    }
-                }
+            let nodes = PathRouter.getChildReadNodes(originalPath, { onlyOwnNodes: true });
+            if (nodes.nodes.length > 0 && nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
+                // If we're the self authority of it, it's not going to change, so we can just cache it safely. 
+                this.parentsSynced.set(originalPath, true);
+                return true;
             }
         }