querysub 0.406.0 → 0.408.0

package/src/0-path-value-core/hackedPackedPathParentFiltering.ts

@@ -1,77 +1,41 @@
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getPathDepth, getPathIndexAssert, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath } from "../path";
-import { PathRouter } from "./PathRouter";
+import { AuthoritySpec, PathRouter } from "./PathRouter";
 import { cache } from "socket-function/src/caching";
 import { authorityLookup } from "./AuthorityLookup";
 
-// NOTE: This code has been moved into PathRouter
-// /** Returns a number between 0 (inclusive) and 1 (exclusive)
-//  *      - See matchesParentRangeFilter, matchesParentRangeFilterPart, and filterChildPaths.
-//  */
-// export function __getRoutingHash(key: string): number {
-//     // Using fastHash is about twice as fast as sha256 (although that might be due to the buffer allocation?)
-//     let hash = fastHash(key);
-//     return hash % (1000 * 1000 * 1000) / (1000 * 1000 * 1000);
-//     // let hash = sha256(key);
-//     // return getBufferFraction(Buffer.from(hash, "hex"));
-// }
 
-let isPrefix = cache((path: string) => {
-    let spec = authorityLookup.getOurSpec();
-    if (!spec) return false;
-    for (let prefix of spec.prefixes) {
-        if (path.startsWith(prefix)) return true;
-    }
-    return false;
-});
+let getSpecForChildPath = (path: string) => authorityLookup.getOurSpec();
+export function registerGetSpecForChildPath(fnc: (path: string) => AuthoritySpec) {
+    getSpecForChildPath = fnc;
+}
 
-// NOTE: Assumes fullPath is already a child of parentPath, and only checks for hash
 export function matchesParentRangeFilter(config: {
     parentPath: string;
     fullPath: string;
     packedPath: string;
 }) {
-    if (config.packedPath === config.packedPath) return true;
+    // If it equals the non packed path, then it must not be packed, and so it must match. 
+    if (config.parentPath === config.packedPath) return true;
     let filter = decodeParentFilter(config.packedPath);
     if (!filter) return true;
-    if (filter.useFullPathHash) {
-        let route = PathRouter.getSingleKeyRoute(config.fullPath);
-        return filter.start <= route && route < filter.end;
-    }
-    let route = PathRouter.getRouteChildKey(config.fullPath);
+    if (filter.start <= 0 && filter.end >= 1) return true;
+    let route = PathRouter.getRouteFull({ path: config.fullPath, spec: getSpecForChildPath(config.fullPath) });
     return filter.start <= route && route < filter.end;
 }
-// export function matchesParentRangeFilterPart(config: {
-//     part: string;
-//     start: number;
-//     end: number;
-// }) {
-//     if (config.start === 0 && config.end === 1) return true;
-//     let hash = __getRoutingHash(config.part);
-//     return config.start <= hash && hash < config.end;
-// }
 
 export const filterChildPathsBase = measureWrap(
     function filterChildPathsBase(parentPath: string, packedSuffix: string, paths: Set<string>): Set<string> {
-        let [startFractionStr, endFractionStr, useFullPathHash] = packedSuffix.split("|");
+        let [startFractionStr, endFractionStr] = packedSuffix.split("|");
         let startFraction = Number(startFractionStr);
         let endFraction = Number(endFractionStr);
 
-        let depth = getPathDepth(parentPath);
-
         let filtered = new Set<string>();
         for (let path of paths) {
-            if (useFullPathHash) {
-                let route = PathRouter.getSingleKeyRoute(path);
-                if (startFraction <= route && route < endFraction) {
-                    filtered.add(path);
-                }
-            } else {
-                let key = getPathIndexAssert(path, depth);
-                let hash = PathRouter.getSingleKeyRoute(key);
-                if (startFraction <= hash && hash < endFraction) {
-                    filtered.add(path);
-                }
+            // TODO: We can make this significantly more efficient as once we know if it's a prefix or not, we know the underlying function call every time. However, that starts to get complicated, and I don't think this is going to be a bottleneck.
+            let route = PathRouter.getRouteFull({ path, spec: getSpecForChildPath(path) });
+            if (startFraction <= route && route < endFraction) {
+                filtered.add(path);
             }
         }
         //console.log(`Filtered ${paths.size} paths to ${filtered.size} paths`);
@@ -83,15 +47,14 @@ export function encodeParentFilter(config: {
     path: string;
     startFraction: number;
     endFraction: number;
-    useFullPathHash?: boolean;
 }) {
-    return hack_setPackedPathSuffix(config.path, `${config.startFraction}|${config.endFraction}|${config.useFullPathHash ? "F" : ""}`);
+    return hack_setPackedPathSuffix(config.path, `${config.startFraction}|${config.endFraction}`);
 }
-export function decodeParentFilter(path: string): { path: string; start: number, end: number, useFullPathHash?: boolean } | undefined {
+export function decodeParentFilter(path: string): { path: string; start: number, end: number } | undefined {
     let packedSuffix = hack_getPackedPathSuffix(path);
     if (!packedSuffix) return undefined;
-    let [startStr, endStr, useFullPathHash] = packedSuffix.split("|");
+    let [startStr, endStr] = packedSuffix.split("|");
     let unpackedPath = hack_stripPackedPath(path);
-    return { path: unpackedPath, start: Number(startStr), end: Number(endStr), useFullPathHash: !!useFullPathHash };
+    return { path: unpackedPath, start: Number(startStr), end: Number(endStr) };
 
 }

package/src/0-path-value-core/pathValueArchives.ts

@@ -12,7 +12,7 @@ import { NodeIdParts, decodeNodeId, decodeNodeIdAssert, encodeNodeId } from "../
 import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
 import { devDebugbreak, isNoNetwork } from "../config";
 import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
-import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { AuthoritySpec, PathRouter, debugSpec } from "./PathRouter";
 import { authorityLookup } from "./AuthorityLookup";
 
 export const archives = lazy(() => wrapArchivesWithCache(getArchives("path-values/")));
@@ -209,7 +209,7 @@ export class PathValueArchives {
 
     @measureFnc
     public async loadValues(authority: AuthoritySpec): Promise<PathValueSnapshot> {
-        console.log(blue(`Loading data at ${new Date().toISOString()}`), { authority });
+        console.log(blue(`Loading data at ${new Date().toISOString()}`), { authority: debugSpec(authority) });
 
         // let testFileName = getOwnNodeId() + "_" + Date.now() + "_" + 1_000_000_000;
         // await archives().assertPathValid(dirName + "data_" + testFileName + ".data");
@@ -223,6 +223,7 @@ export class PathValueArchives {
         let totalSize = 0;
 
         let dataPaths: string[] = [];
+        let allPaths: string[] = [];
         let readCache = new Map<string, Buffer>();
 
         let downloadStartTime = Date.now();
@@ -233,8 +234,12 @@ export class PathValueArchives {
             //  the source file, but by now it has moved, so we don't see either file, even though at all times
             //  both files existed).
             let time = Date.now();
-            dataPaths = await this.getValuePaths(authority);
-            console.log(green(`${dataPaths.length} data paths in ${formatTime(Date.now() - time)}`));
+            let valueObjs = await this.getValuePaths(authority);
+            dataPaths = valueObjs.pickedPaths;
+            allPaths = valueObjs.allPaths;
+            let allValueCount = allPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
+            let pickedValueCount = dataPaths.reduce((acc, path) => acc + this.decodeDataPath(path).valueCount, 0);
+            console.log(green(`${dataPaths.length}/${allPaths.length} (values ${formatNumber(pickedValueCount)} / ${formatNumber(allValueCount)}) data paths in ${formatTime(Date.now() - time)}`));
             // NOTE: If the notMatched count is high enough, it is possible NodePathAuthorities.ts:authoritiesMightOverlap is
             //  too loose, and should be removing more cases.
 
@@ -318,7 +323,7 @@ export class PathValueArchives {
 
         let relevantCount = totalCount - notMatched;
         parseTime = Date.now() - parseTime;
-        console.log((`${green("Finished parsing")} ${formatNumber(totalCount)} in ${formatNumber(totalSize)}B (${formatNumber(uniqueCount / relevantCount * 100)}% unique paths = ${formatNumber(uniqueCount)}), in ${green(formatTime(parseTime))}, ${fileCount} files, (${formatNumber(notMatched / totalCount * 100)}% ignored, due to being for another authority) at ${new Date().toISOString()}. Startup time ${green(formatTime(process.uptime() * 1000))}`), { authority });
+        console.log((`${green("Finished parsing")} ${formatNumber(totalCount)} in ${formatNumber(totalSize)}B (${formatNumber(uniqueCount / relevantCount * 100)}% unique paths = ${formatNumber(uniqueCount)}), in ${green(formatTime(parseTime))}, ${fileCount} files, (${formatNumber(notMatched / totalCount * 100)}% ignored, due to being for another authority) at ${new Date().toISOString()}. Startup time ${green(formatTime(process.uptime() * 1000))}`), { authority: debugSpec(authority) });
 
         if (tooOldValues) {
             console.warn(yellow(`Discarded ${formatNumber(tooOldValues)} values discarded for being part of hanging server writes`));
@@ -437,7 +442,10 @@ export class PathValueArchives {
     }
 
     @measureFnc
-    public async getValuePaths(authority: AuthoritySpec): Promise<string[]> {
+    public async getValuePaths(authority: AuthoritySpec): Promise<{
+        pickedPaths: string[];
+        allPaths: string[];
+    }> {
         // let paths: string[] = [];
         // let pickedDirectories = await this.getAuthorityDirs(authority);
 
@@ -450,8 +458,11 @@ export class PathValueArchives {
 
         let locker = await this.getArchiveLocker();
         let allFiles = (await locker.getAllValidFiles()).map(x => x.file);
-        let doesMatch = cache((dir: string) => PathRouter.overlapsPathIdentifier(authority, dir));
-        return allFiles.filter(x => doesMatch(x.split("/").slice(0, -1).join("/")));
+        let pickedPaths = allFiles.filter(x => PathRouter.overlapsPathIdentifier(authority, x));
+        return {
+            pickedPaths,
+            allPaths: allFiles,
+        };
     }
     @measureFnc
     public async getValuePathSizes(paths: string[]): Promise<number[]> {

package/src/0-path-value-core/pathValueCore.ts

@@ -23,7 +23,9 @@ import { ellipsisMiddle } from "../misc";
 import { fastHash } from "../misc/hash";
 import { authorityLookup } from "./AuthorityLookup";
 import { onPathInteracted } from "../diagnostics/pathAuditerCallback";
-import { filterChildPathsBase } from "./hackedPackedPathParentFiltering";
+import { decodeParentFilter, filterChildPathsBase } from "./hackedPackedPathParentFiltering";
+import { isDiskAudit } from "../config";
+import { removeRange } from "../rangeMath";
 
 setImmediate(async () => {
     // Import everything will dynamically import, so the client side can tell that it's required. 
@@ -401,8 +403,8 @@ class AuthorityPathValueStorage {
     private isSyncedCache = registerResource("paths|isSyncedCache", new Set<string>());
     private parentsSynced = registerResource("paths|parentsSynced", new Map<string,
         true |
-        // If set, contains the nodes we have yet to sync before we can consider the parent synced.
-        Set<string>
+        // If set, contains the ranges received so far for this parent path.
+        { start: number; end: number }[]
     >());
 
     public DEBUG_UNWATCH = false;
@@ -443,9 +445,10 @@ class AuthorityPathValueStorage {
         // Include all values with other.time <= time, returning the one with the highest time
         //      (either that is valid, or, if readInvalid is true, just the highest time).
         time?: Time,
-        readInvalid = false
+        readInvalid = false,
+        noAudit?: "noAudit"
     ): PathValue | undefined {
-        if (!this.isEventPath(path)) {
+        if (!this.isEventPath(path) && !noAudit) {
             onPathInteracted(path, 0);
         }
         let overrideValue: PathValue | undefined;
@@ -542,6 +545,9 @@ class AuthorityPathValueStorage {
         if (this.DEBUG_UNWATCH) {
             console.log(blue(`Unsyncing path at ${Date.now()}`), path);
         }
+        if (isDiskAudit()) {
+            auditLog("DESTROY PATH", { path });
+        }
 
         this.isSyncedCache.delete(path);
         this.removePathFromStorage(path, "unwatched");
@@ -603,10 +609,45 @@ class AuthorityPathValueStorage {
         return false;
     }
     public isParentSynced(path: string) {
-        // We have to check a test child path to see if we are a self authority
-        if (PathRouter.isSelfAuthority(appendToPathStr(path, ""))) return true;
+        let range = decodeParentFilter(path);
+        path = hack_stripPackedPath(path);
+        if (PathRouter.isLocalPath(path)) return true;
+
+        let synced = this.parentsSynced.get(path);
+        if (synced === true) return true;
+        // See if the ranges received so far cover the requested range. If we only request a partial range, then this will always be the case. We'll never fully synchronize the path. 
+        if (synced && range) {
+            let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
+            for (let range of synced) {
+                removeRange(missingRanges, range);
+            }
+            if (missingRanges.length === 0) return true;
+        }
 
-        return this.parentsSynced.get(path) === true;
+        // Self authority check
+        {
+            let nodes = PathRouter.getChildReadNodes(path, { onlyOwnNodes: true });
+            if (nodes.nodes.length > 0) {
+                if (nodes.nodes.every(x => isOwnNodeId(x.nodeId))) {
+                    // If it's not a partial 30 and we're self authority, we're always going to be self authority, so we can just cache it in the lookup. 
+                    this.parentsSynced.set(path, true);
+                    return true;
+                }
+                // Might be a partial authority
+                if (range) {
+                    let selfNodes = nodes.nodes.filter(x => isOwnNodeId(x.nodeId));
+                    let missingRanges: { start: number; end: number }[] = [{ start: range.start, end: range.end }];
+                    for (let range of selfNodes) {
+                        removeRange(missingRanges, range.range);
+                    }
+                    if (missingRanges.length === 0) {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        return false;
     }
 
     /** Obviously just for debugging. Doesn't trigger any watchers, just erases it as if we never had it. There shouldn't be really any downstream caches within the same process at least, and so this should result in the next read having the value missing. If we in the future add downstream caches, we're going to have to invalidate them as well.
@@ -622,31 +663,37 @@ class AuthorityPathValueStorage {
         // this.isSyncedCache.delete(path);
     }
 
-    private getMultiNodesForParent: (path: string) => Map<string, unknown> | undefined = () => undefined;
-    public setGetMultiNodesForParent(fnc: (path: string) => Map<string, unknown> | undefined) {
-        this.getMultiNodesForParent = fnc;
-    }
-
     public addParentSyncs(parentSyncs: { parentPath: string; sourceNodeId: string }[]) {
-
         for (let obj of parentSyncs) {
-            let parentPath = obj.parentPath;
-            let sourceNodeId = obj.sourceNodeId;
+            let decoded = decodeParentFilter(obj.parentPath);
+            let range = decoded ? { start: decoded.start, end: decoded.end } : { start: 0, end: 1 };
+            let parentPath = hack_stripPackedPath(obj.parentPath);
+
             let prevSynced = this.parentsSynced.get(parentPath);
             if (prevSynced === true) continue;
-            let remoteNodeIds = this.getMultiNodesForParent(parentPath);
-            // NOTE: If there is only 1 authority, assume we received data from it, as who else would we receive it from?
-            if (!remoteNodeIds || remoteNodeIds.size === 1) {
-                this.parentsSynced.set(parentPath, true);
+
+            let ranges: { start: number; end: number }[];
+            if (!prevSynced) {
+                ranges = [];
+                this.parentsSynced.set(parentPath, ranges);
             } else {
-                if (!prevSynced) {
-                    prevSynced = new Set(remoteNodeIds.keys());
-                    this.parentsSynced.set(parentPath, prevSynced);
-                }
-                prevSynced.delete(sourceNodeId);
-                if (prevSynced.size === 0) {
-                    this.parentsSynced.set(parentPath, true);
+                ranges = prevSynced;
+            }
+
+            ranges.push(range);
+
+            function rangesCoverFull(ranges: { start: number; end: number }[]): boolean {
+                let sorted = ranges.slice().sort((a, b) => a.start - b.start);
+                let covered = 0;
+                for (let range of sorted) {
+                    if (range.start > covered) return false;
+                    if (range.end > covered) covered = range.end;
                 }
+                return covered >= 1;
+            }
+
+            if (rangesCoverFull(ranges)) {
+                this.parentsSynced.set(parentPath, true);
             }
         }
     }
@@ -1044,6 +1091,7 @@ class AuthorityPathValueStorage {
         let { spec, startTime, endTime } = config;
         let result: PathValue[] = [];
         for (let [path, values] of this.values) {
+            if (PathRouter.isLocalPath(path)) continue;
             if (PathRouter.matchesAuthoritySpec(spec, path)) {
                 for (let value of values) {
                     let time = value.time.time;

package/src/0-path-value-core/startupAuthority.ts

@@ -1,7 +1,7 @@
 import { delay } from "socket-function/src/batching";
 import { DISK_FLUSH_INTERVAL } from "../diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants";
 import { authorityLookup } from "./AuthorityLookup";
-import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { AuthoritySpec, PathRouter, debugSpec } from "./PathRouter";
 import { PathValueControllerBase, pathValueCommitter } from "./PathValueController";
 import { validStateComputer } from "./ValidStateComputer";
 import { MAX_TIME_UNTIL_DISK_FLUSH, authorityStorage, pathValueArchives } from "./pathValueCore";
@@ -11,7 +11,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 const MAX_LOAD_RETRIES = 3;
 export async function startupAuthority(spec: AuthoritySpec) {
     let startTime = Date.now();
-    console.log(`Becoming authority for`, { spec });
+    console.log(`Becoming authority for`, { spec: debugSpec(spec) });
 
     // NOTE: This will broadcast all the other servers about our spec, and so all other servers will start giving us live writes.
     await authorityLookup.setOurSpec(spec);
@@ -20,23 +20,23 @@ export async function startupAuthority(spec: AuthoritySpec) {
     let readEndTime = Number.MAX_SAFE_INTEGER;
 
     let sources = await PathRouter.getAuthoritySources({ target: spec });
-    console.log(`Found sources when finding spec, count: ${sources.length}`, { spec, sources });
+    console.log(`Found sources when finding spec, count: ${sources.length}`, { us: debugSpec(spec), sources });
     async function loadFromSourceBase(source: AuthoritySpec) {
-        console.log(blue(`Loading values from source ${source.nodeId}`), { nodeId: source.nodeId, spec: source, readStartTime, readEndTime });
+        console.log(blue(`Loading values from source ${source.nodeId}`), { nodeId: source.nodeId, spec: debugSpec(source), readStartTime, readEndTime });
         let values = await PathValueControllerBase.getInitialValues({
             nodeId: source.nodeId,
             spec: source,
             startTime: readStartTime,
             endTime: readEndTime,
         });
-        console.log(blue(`Finished loading values from source ${source.nodeId}, found ${formatNumber(values.length)} values`), { nodeId: source.nodeId, spec: source, values: values.length });
+        console.log(blue(`Finished loading values from source ${source.nodeId}, found ${formatNumber(values.length)} values`), { nodeId: source.nodeId, spec: debugSpec(source), values: values.length });
         validStateComputer.ingestValuesAndValidStates({
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
             doNotArchive: true,
         });
-        console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: source, values: values.length });
+        console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: debugSpec(source), values: values.length });
     }
     async function loadFromSource(source: AuthoritySpec, retries = 3) {
         try {
@@ -60,15 +60,15 @@ export async function startupAuthority(spec: AuthoritySpec) {
     let diskPromise = (async () => {
         let snapshot = await pathValueArchives.loadValues(spec);
         let flat = Object.values(snapshot.values).flat();
-        console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec, values: flat.length });
+        console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
         // NOTE: If it's on disk, it can't be rejected, so there's no point in passing it through our validStateComputer (and passing it through is VERY slow).
         authorityStorage.ingestValues(flat, { doNotArchive: true });
-        console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec, values: flat.length });
+        console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
     })();
 
     await Promise.all([...sourcePromises, diskPromise]);
 
     await authorityLookup.setIsReady();
     let duration = Date.now() - startTime;
-    console.log(green(`Finished becoming authority, took ${formatTime(duration)}`), { spec });
+    console.log(green(`Finished becoming authority, took ${formatTime(duration)}`), { spec: debugSpec(spec) });
 }