querysub 0.427.0 → 0.429.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.427.0",
+  "version": "0.429.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/AuthorityLookup.ts

@@ -18,8 +18,8 @@ import { getAllAuthoritySpec, getEmptyAuthoritySpec } from "./PathRouterServerAu
 
 setImmediate(() => import("../3-path-functions/syncSchema"));
 
-let NETWORK_POLL_INTERVAL = timeInMinute * 5;
-let CALL_TIMEOUT = isPublic() ? timeInSecond * 20 : timeInSecond * 3;
+let NETWORK_POLL_INTERVAL = timeInMinute * 1;
+let CALL_TIMEOUT = timeInSecond * 5;
 
 export type AuthorityEntry = {
     nodeId: string;

package/src/0-path-value-core/PathRouter.ts

@@ -10,6 +10,7 @@ import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
 import { sha256 } from "js-sha256";
 import { rangesOverlap, removeRange } from "../rangeMath";
+import { decodeParentFilter } from "./hackedPackedPathParentFiltering";
 
 
 // Cases
@@ -57,6 +58,19 @@ function matchesPrefix(matcher: PrefixMatcher, path: string): boolean {
             return false;
         }
     }
+    if (getPathIndex(path, matcher.childKeyIndex) === undefined) return false;
+    return true;
+}
+// Checks if they're asking for the exact prefix, which would mean that the children would be the direct children of this prefix matcher. 
+function isPrefixParent(matcher: PrefixMatcher, path: string): boolean {
+    if (!path.startsWith(matcher.prefix)) return false;
+    let partsA = getPathFromStr(matcher.originalPrefix);
+    let partsB = getPathFromStr(path);
+    for (let i = 0; i < partsA.length; i++) {
+        // Empty string is wildcard
+        if (partsA[i] === "") continue;
+        if (partsB[i] !== partsA[i]) return false;
+    }
     return true;
 }
 /** Empty string path part becomes a wildcard */
@@ -535,11 +549,17 @@ export class PathRouter {
             range: { start: number; end: number };
         }[];
     } {
+        let parentRange = decodeParentFilter(path) || {
+            start: 0,
+            end: 1,
+        };
+        path = hack_stripPackedPath(path);
         let preferredNodeIds = new Set(config?.preferredNodeIds ?? []);
 
         // If a prefix is a parent of path, then it is the same as matching just the path directly
         // (If our prefix directly equals one of the other matches, then it's more complicated, As then, the child keys of path are what is hashed, and so all the children will have different routes, so we might match multiple nodes. The same thing if we're matching the remaining case, in which case it's a full path hash, so the child key matters, and again, different routes).
         //      - The different route case is how the FuntionRunner works, and without it large databases couldn't run functions. However, most applications won't directly use it.
+        // NOTE: The only own nodes flag is actually so we can access this before the topology finishes synchronizing. Because we want to be able to call get child read nodes from path value core for some really basic stuff. It also is what the caller wants, but that's not a good enough reason to add this check. The reason we have this check is because without it, the topology won't have finished synchronizing and startup won't work.
         let allSources = config?.onlyOwnNodes ? [{ nodeId: getOwnNodeId(), authoritySpec: authorityLookup.getOurSpec() }] : authorityLookup.getTopologySync();
         // Prefer our own node
         sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
@@ -549,14 +569,13 @@ export class PathRouter {
 
 
         // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
-        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => y.prefix === path)).map(x => x.authoritySpec);
+        let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => isPrefixParent(y, path))).map(x => x.authoritySpec);
         if (hasPrefix.length > 0) {
             shuffle(hasPrefix, Math.random());
             sort(hasPrefix, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
 
             let missingRanges: { start: number; end: number }[] = [{
-                start: 0,
-                end: 1,
+                ...parentRange
             }];
             let usedParts: {
                 nodeId: string;
@@ -585,7 +604,7 @@ export class PathRouter {
         let nestedMatches = allSources.filter(x => {
             // There's nested prefixes, so if we match any prefix explicitly, we can't just take one of the previous prefixes because that isn't how the hashing will work. 
             // - This happens if it's a direct match, but one of the shards is down, in which case we can't get a full match.
-            if (x.authoritySpec.prefixes.some(y => y.prefix === path)) return false;
+            if (x.authoritySpec.prefixes.some(y => isPrefixParent(y, path))) return false;
 
             // If our path, which we're going to read the children of, is the child of another path, then it means in that other path, the child key will be known to us constant, and so we're going to match exactly one authority. 
             return (
@@ -617,8 +636,7 @@ export class PathRouter {
             sort(fullPathMatches, x => preferredNodeIds.has(x.nodeId) ? -1 : 1);
             sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
             let missingRanges: { start: number; end: number }[] = [{
-                start: 0,
-                end: 1,
+                ...parentRange
             }];
             let usedParts: {
                 nodeId: string;
@@ -645,8 +663,6 @@ export class PathRouter {
         }
 
 
-
-
         if (!config?.onlyOwnNodes) {
             // TODO: We could maybe match a partial match. However, even that is suspect. The site being partially broken is almost worse than it being completely broken. We should just get ALL the shards running again...
 

package/src/0-path-value-core/pathValueCore.ts

@@ -609,11 +609,12 @@ class AuthorityPathValueStorage {
         return false;
     }
     public isParentSynced(path: string) {
+        let originalPath = path;
         let range = decodeParentFilter(path);
         path = hack_stripPackedPath(path);
         if (PathRouter.isLocalPath(path)) return true;
 
-        let synced = this.parentsSynced.get(path);
+        let synced = this.parentsSynced.get(originalPath) || this.parentsSynced.get(path);
         if (synced === true) return true;
         // See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path. 
         if (synced && range) {
@@ -626,24 +627,11 @@ class AuthorityPathValueStorage {
 
         // Self authority check
         {
-            let nodes = PathRouter.getChildReadNodes(path, { onlyOwnNodes: true });
-            if (nodes.nodes.length > 0) {
-                if (nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
-                    // If it's not a partial 30 and we're self authority, we're always going to be self authority, so we can just cache it in the lookup. 
-                    this.parentsSynced.set(path, true);
-                    return true;
-                }
-                // Might be a partial authority
-                if (range) {
-                    let selfNodes = nodes.nodes.filter(x => isOwnNodeId(x.nodeId));
-                    let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
-                    for (let range of selfNodes) {
-                        removeRange(missingRanges, range.range);
-                    }
-                    if (missingRanges.length === 0) {
-                        return true;
-                    }
-                }
+            let nodes = PathRouter.getChildReadNodes(originalPath, { onlyOwnNodes: true });
+            if (nodes.nodes.length > 0 && nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
+                // If we're the self authority of it, it's not going to change, so we can just cache it safely. 
+                this.parentsSynced.set(originalPath, true);
+                return true;
             }
         }
 
@@ -1121,4 +1109,5 @@ setImmediate(() => {
 export { pathValueArchives };
 
 (globalThis as any).core = module.exports;
+(globalThis as any).authorityStorage = authorityStorage;
 (globalThis as any).SocketFunction = SocketFunction;

package/src/1-path-client/RemoteWatcher.ts

@@ -486,7 +486,6 @@ export class RemoteWatcher {
                             auditLog("Asking to watch parent path", { path, authorityId, targetNodeThreadId: debugNodeThread(authorityId) });
                         }
                     }
-
                     logErrors(RemoteWatcher.REMOTE_WATCH_FUNCTION(config, authorityId));
                 }
             });