querysub 0.405.0 → 0.406.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.405.0",
+  "version": "0.406.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathRouter.ts

@@ -32,10 +32,7 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
-    prefixes: {
-        prefix: string;
-        hashIndex: number;
-    }[];
+    prefixes: string[];
     excludeDefault?: boolean;
 };
 
@@ -68,13 +65,11 @@ export class PathRouter {
             return override.route;
         }
 
-        let prefix = spec.prefixes.find(x => path.startsWith(x.prefix) && x.prefix !== path);
+        let prefix = spec.prefixes.find(x => path.startsWith(x) && x !== path);
         if (prefix) {
-            let key = getPathIndex(path, prefix.hashIndex);
+            let key = getPathIndex(path, getPathDepth(prefix));
             if (key === undefined) {
-                require("debugbreak")(2);
-                debugger;
-                throw new Error(`Impossible, hash index ${prefix.hashIndex} is out of range for path ${path}, but it matched the prefix ${prefix.prefix}`);
+                throw new Error(`Impossible, hash index ${getPathDepth(prefix)} is out of range for path ${path}, but it matched the prefix ${prefix}`);
             }
             return this.getSingleKeyRoute(key);
         }
@@ -112,7 +107,15 @@ export class PathRouter {
     private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
-        return ["P", rangeStart.toString(), rangeEnd.toString(), ...prefixes.map(x => this.getPrefixHash(x)), ""].join("!");
+        return [
+            "P",
+            rangeStart.toString(),
+            rangeEnd.toString(),
+            ...prefixes.map(x => this.getPrefixHash(x)),
+            // Add some prefixes. They'll be interpreted as hashes, but it's fine because it's highly unlikely they'll collide with a real hash. AND... It's very useful for debugging what's in which file. 
+            ...prefixes.slice(0, 5).map(x => getLastPathPart(x).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
+            ""
+        ].join("!");
 
     }
     private static decodeIdentifier(identifier: string): { prefixHashes: string[]; rangeStart: number; rangeEnd: number } | "remaining" {
@@ -141,10 +144,10 @@ export class PathRouter {
         }
 
         let prefixes = ourSpec.prefixes.slice();
-        sort(prefixes, x => x.prefix.length);
+        sort(prefixes, x => x.length);
         function getPrefix(path: string): string | undefined {
             for (let prefix of prefixes) {
-                if (path.startsWith(prefix.prefix) && prefix.prefix !== path) return prefix.prefix;
+                if (path.startsWith(prefix) && prefix !== path) return prefix;
             }
             return undefined;
         }
@@ -168,9 +171,6 @@ export class PathRouter {
         }));
         sort(prefixGroups, x => -x.values.length);
 
-        require("debugbreak")(2);
-        debugger;
-
         let groups: {
             prefixes: string[];
             values: PathValue[][];
@@ -261,7 +261,7 @@ export class PathRouter {
         // Match all remaining, as we don't store enough information to know what prefixes they excluded
         if (decodeObj === "remaining") return true;
 
-        let ourHashes = authority.prefixes.map(x => this.getPrefixHash(x.prefix));
+        let ourHashes = authority.prefixes.map(x => this.getPrefixHash(x));
         // If it pulled off some values as prefixes, but we did full path hashes, then the hashes are going to be totally different. And so there could easily be overlap.
         if (decodeObj.prefixHashes.some(x => !ourHashes.includes(x))) return true;
         // However, if we hash everything the same, then the overlap is purely a case of if the route ranges overlap. 
@@ -305,12 +305,12 @@ export class PathRouter {
 
         let groupByPrefixHash = new Map<string, AuthoritySpec[]>();
         for (let source of allSources) {
-            let usedPrefixes = source.authoritySpec.prefixes.map(x => x.prefix);
+            let usedPrefixes = source.authoritySpec.prefixes;
 
             if (target.excludeDefault) {
                 // This relaxes the restrictions, as with no default hashes, it means if a value isn't in one of our prefixes, even if it's inconsistent in the sources, it's fine. 
                 // (And... Otherwise, we can't filter these at all because otherwise the default/remaining group will be different, even if it's a superset/subset relationship)
-                let targetUsedPrefixes = new Set(target.prefixes.map(x => x.prefix));
+                let targetUsedPrefixes = new Set(target.prefixes);
                 usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x));
             }
             let prefixHash = sha256(JSON.stringify(usedPrefixes.sort()));
@@ -329,17 +329,17 @@ export class PathRouter {
         }
 
         function hashesSameAsTarget(spec: AuthoritySpec): boolean {
-            let targetUsedPrefixes = new Set(target.prefixes.map(x => x.prefix));
+            let targetUsedPrefixes = new Set(target.prefixes);
             if (target.excludeDefault) {
                 // If target is excluding default, then it doesn't matter if the spec is including it. We're only looking for matching values in target.
-                return spec.prefixes.every(x => targetUsedPrefixes.has(x.prefix));
+                return spec.prefixes.every(x => targetUsedPrefixes.has(x));
             }
             // if !targetExcludeDefault, then !spec.excludeDefault, because we filtered out other cases earlier (so we don't need to handle spec.
 
             if (spec.prefixes.length !== target.prefixes.length) return false;
             // Otherwise, All of the prefixes have to be identical
             for (let prefix of spec.prefixes) {
-                if (!targetUsedPrefixes.has(prefix.prefix)) return false;
+                if (!targetUsedPrefixes.has(prefix)) return false;
             }
             return true;
         }
@@ -440,7 +440,7 @@ export class PathRouter {
         let allSources = authorityLookup.getTopologySync();
         allSources = allSources.filter(x => !isOwnNodeId(x.nodeId));
         let nestedMatches = allSources.filter(x =>
-            x.authoritySpec.prefixes.some(y => path.startsWith(y.prefix) && y.prefix !== path)
+            x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
             && this.matchesAuthoritySpec(x.authoritySpec, path)
         );
         if (nestedMatches.length > 0) {
@@ -459,10 +459,7 @@ export class PathRouter {
         let fullSources = this.getAuthoritySources({
             target: {
                 nodeId: "",
-                prefixes: [{
-                    prefix: path,
-                    hashIndex: getPathDepth(path),
-                }],
+                prefixes: [path],
                 routeStart: 0,
                 routeEnd: 1,
                 excludeDefault: true,

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -62,7 +62,7 @@ export const hasPrefixHash = cacheLimited(1000 * 10,
     (config: { spec: AuthoritySpec, prefixHash: string }) => {
         let { spec, prefixHash } = config;
         for (let prefix of spec.prefixes) {
-            if (getPrefixHash(prefix.prefix) === prefixHash) {
+            if (getPrefixHash(prefix) === prefixHash) {
                 return true;
             }
         }

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -20,10 +20,7 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
                 nodeId: "",
                 routeStart: 0,
                 routeEnd: 1,
-                prefixes: prefixes.map(prefix => ({
-                    prefix,
-                    hashIndex: getPathDepth(prefix),
-                })),
+                prefixes: prefixes,
             };
         }
         return undefined;
@@ -46,10 +43,7 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
         nodeId: "",
         routeStart: rangeStart,
         routeEnd: rangeEnd,
-        prefixes: usePrefixes.map(prefix => ({
-            prefix,
-            hashIndex: getPathDepth(prefix),
-        })),
+        prefixes: usePrefixes,
         excludeDefault: excludeDefault || undefined,
     };
 }

package/src/0-path-value-core/hackedPackedPathParentFiltering.ts

@@ -20,7 +20,7 @@ let isPrefix = cache((path: string) => {
     let spec = authorityLookup.getOurSpec();
     if (!spec) return false;
     for (let prefix of spec.prefixes) {
-        if (path.startsWith(prefix.prefix)) return true;
+        if (path.startsWith(prefix)) return true;
     }
     return false;
 });

package/src/2-proxy/garbageCollection.ts

@@ -19,6 +19,7 @@ import { getStorageDir, getSubFolder } from "../fs";
 import fs from "fs";
 import { sha256 } from "js-sha256";
 import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 
 
 export interface AliveChecker<T = unknown> {
@@ -34,7 +35,7 @@ export interface AliveChecker<T = unknown> {
      *  NOTE: The value might only have a single heavily nested value, not being a valid object. If the object
      *      lookups broken, probably just return false, to clean up the straggling values.
      */
-    isAlive: (value: T, path: string[]) => boolean | unknown;
+    isAlive: (value: T, path: string[], valuePath: PathValue | undefined) => boolean | unknown;
 
     /** Alias for lockBasedDelete & unsafeLiveReads */
     lock?: boolean;
@@ -254,7 +255,7 @@ function sandboxedIsAlive(config: {
             proxy = proxy[part];
         }
         try {
-            return !!atomicRaw(checker.isAlive(proxy, path));
+            return !!atomicRaw(checker.isAlive(proxy, path, undefined));
         } catch {
             config.errors.count++;
             return true;
@@ -308,7 +309,7 @@ function sandboxedGetLocks(config: {
 
     let evalResult = getWatchEvaluator()(() => {
         let value = proxyWatcher.getCallbackPathValue(pathStr);
-        return !!atomic(checker.isAlive(value, path));
+        return !!atomic(checker.isAlive(pathValueSerializer.getPathValue(value), path, value));
     }, { getReadLocks: true });
 
     return evalResult.readLocks!;

package/src/functional/promiseCache.ts

@@ -34,7 +34,10 @@ export function cacheAsyncLimited<Arg, Return>(limit: number, fnc: (arg: Arg) =>
 export const cacheAsyncSynced = cacheAsyncLimitedJSON;
 export function cacheAsyncLimitedJSON<Arg, Return>(
     limit: number,
-    fnc: (arg: Arg) => Promise<Return>
+    fnc: (arg: Arg) => Promise<Return>,
+    config?: {
+        notReadyValue?: Return;
+    }
 ): ((arg: Arg) => Return | undefined) & {
     clear(): void;
     promise: (arg: Arg) => Promise<Return>;
@@ -71,6 +74,6 @@ export function cacheAsyncLimitedJSON<Arg, Return>(
             }
         );
         proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: fnc.toString() });
-        return undefined;
+        return config?.notReadyValue;
     };
 }

package/src/path.ts

@@ -181,8 +181,12 @@ export function removePathLastPart(pathStr: string) {
     return pathStr.slice(0, getStartOfLastPart(pathStr));
 }
 
+let lastPastPath = { path: "", depth: 0 };
 /** getPathDepth(pathStr) === getPathFromStr(pathStr).length */
 export function getPathDepth(pathStr: string): number {
+    if (lastPastPath.path === pathStr) {
+        return lastPastPath.depth;
+    }
     // Start at 1, to skip the first pathDelimit
     let index = 1;
     // Start at -1, as the last loop doesn't have pathDelimitEscaped, but is still incremented
@@ -191,6 +195,8 @@ export function getPathDepth(pathStr: string): number {
         count++;
         index = pathStr.indexOf(pathDelimitEscaped, index) + 1;
     }
+    lastPastPath.path = pathStr;
+    lastPastPath.depth = count;
     return count;
 }
 

package/test.ts

@@ -25,8 +25,6 @@ import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
 async function main() {
     let prefixes = await getShardPrefixes();
     console.log(prefixes);
-    let deployPath = path.resolve("./deploy.ts");
-    await import(deployPath);
     // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
     // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
     // for (let path of paths) {
@@ -60,7 +58,7 @@ async function main() {
     //     return path.split("/").at(-1)!;
     // }
 
-    /*
+    //*
     let allValuesObj = await pathValueArchives.loadValues({
         nodeId: "",
         prefixes: [],
@@ -69,7 +67,7 @@ async function main() {
     });
     let allValues = Object.values(allValuesObj.values).flat();
     console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
-    */
+    //*/
 
     // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
 

package/src/diagnostics/logs/BrowserLargeFileCache.ts

@@ -1,64 +0,0 @@
-import { SocketFunctionClientHook } from "socket-function/SocketFunctionTypes";
-import { cache, lazy } from "socket-function/src/caching";
-import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
-import { sha256 } from "js-sha256";
-import { errorToUndefined } from "../../errors";
-import { isNode } from "typesafecss";
-
-let getRootDirectory = lazy(async () => {
-    await navigator.storage.persist();
-    return navigator.storage.getDirectory();
-});
-
-export const getBrowserLargeFileCache = cache((name: string) => new BrowserLargeFileCache(name));
-
-export class BrowserLargeFileCache {
-    constructor(private name: string) { }
-
-    private getDir = lazy(async () => {
-        let root = await getRootDirectory();
-        return await root.getDirectoryHandle(this.name, { create: true });
-    });
-
-    public async set(key: string, value: Buffer) {
-        let dir = await this.getDir();
-        let file = await dir.getFileHandle(key, { create: true });
-        let writable = await file.createWritable();
-        await writable.write(value);
-        await writable.close();
-    }
-    public async get(key: string): Promise<Buffer | undefined> {
-        let dir = await this.getDir();
-        try {
-            let file = await dir.getFileHandle(key, { create: false });
-            let readable = await file.getFile();
-            return Buffer.from(await readable.arrayBuffer());
-        } catch {
-            return undefined;
-        }
-    }
-}
-
-/** Cache key = [args, functionName, classGuid]
- *      - Not nodeId, as that change so frequently, so caching based on server is not usually useful.
- *          - If you want to cache based on nodeId, pass it as an unused arg.
- * 
- *  Uses a file per key, and never cleans them up. So... basically, this sucks, but, it should
- *      work for a few specific functions with large and consistent values.
- */
-export const cacheCalls: SocketFunctionClientHook = async config => {
-    if (isNode()) return;
-
-    let { args, functionName, classGuid } = config.call;
-    let bucket = sha256(JSON.stringify({ functionName, classGuid }));
-    let cache = getBrowserLargeFileCache(bucket);
-    let key = sha256(encodeCborx(args));
-    let cachedValue = await cache.get(key);
-    if (cachedValue) {
-        config.overrideResult = decodeCborx(cachedValue);
-        return;
-    }
-    config.onResult.push(async (result) => {
-        await errorToUndefined(cache.set(key, encodeCborx(result)));
-    });
-};