querysub 0.404.0 → 0.406.0

package/bin/join.js

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
 
 require("typenode");
-require("../src/archiveapps/archiveGCEntry");
+require("../src/archiveapps/archiveJoinEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.404.0",
+  "version": "0.406.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathRouter.ts

@@ -1,5 +1,5 @@
 import { sort } from "socket-function/src/misc";
-import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1 } from "../path";
+import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1, hack_stripPackedPath } from "../path";
 import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
@@ -32,10 +32,7 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
-    prefixes: {
-        prefix: string;
-        hashIndex: number;
-    }[];
+    prefixes: string[];
     excludeDefault?: boolean;
 };
 
@@ -59,18 +56,21 @@ export class PathRouter {
         path: string;
         spec: AuthoritySpec;
     }): number {
-        // NOTE: getSelfPathIdentifierTargets also hardcodes this logic, so it can hash many values quickly
+        // NOTE: getPathIdentifierTargets also hardcodes this logic, so it can hash many values quickly
         let { path, spec } = config;
+        path = hack_stripPackedPath(path);
         let override = getRoutingOverride(path);
         if (override) {
             if (!hasPrefixHash({ spec, prefixHash: override.prefixHash })) return -1;
             return override.route;
         }
 
-        let prefix = spec.prefixes.find(x => path.startsWith(x.prefix) && x.prefix !== path);
+        let prefix = spec.prefixes.find(x => path.startsWith(x) && x !== path);
         if (prefix) {
-            let key = getPathIndex(path, prefix.hashIndex);
-            if (key === undefined) throw new Error(`Impossible, hash index ${prefix.hashIndex} is out of range for path ${path}, but it matched the prefix ${prefix.prefix}`);
+            let key = getPathIndex(path, getPathDepth(prefix));
+            if (key === undefined) {
+                throw new Error(`Impossible, hash index ${getPathDepth(prefix)} is out of range for path ${path}, but it matched the prefix ${prefix}`);
+            }
             return this.getSingleKeyRoute(key);
         }
         if (spec.excludeDefault) return -1;
@@ -107,13 +107,21 @@ export class PathRouter {
     private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
-        return ["P", rangeStart.toString(), rangeEnd.toString(), ...prefixes.map(x => this.getPrefixHash(x))].join("!");
+        return [
+            "P",
+            rangeStart.toString(),
+            rangeEnd.toString(),
+            ...prefixes.map(x => this.getPrefixHash(x)),
+            // Add some prefixes. They'll be interpreted as hashes, but it's fine because it's highly unlikely they'll collide with a real hash. AND... It's very useful for debugging what's in which file. 
+            ...prefixes.slice(0, 5).map(x => getLastPathPart(x).replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 20)),
+            ""
+        ].join("!");
 
     }
     private static decodeIdentifier(identifier: string): { prefixHashes: string[]; rangeStart: number; rangeEnd: number } | "remaining" {
         if (!identifier.startsWith("P")) return "remaining";
         let parts = identifier.split("!");
-        if (parts[1] === "REMAINING") return "remaining";
+        if (parts[1].startsWith("REMAINING")) return "remaining";
         return {
             rangeStart: parseFloat(parts[1]),
             rangeEnd: parseFloat(parts[2]),
@@ -125,7 +133,7 @@ export class PathRouter {
     // NOTE: Encodes all the data, even if it matches our spec or not. If you don't want this, you have to filter first
     //  - If this becomes an issue we COULD filter, as we can do it quickly, but I don't think it is required, as all the present usecases prefilter anyways.
     @measureFnc
-    public static getSelfPathIdentifierTargets(values: PathValue[]): Map<string, PathValue[]> {
+    public static getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Map<string, PathValue[]> {
         // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
         const MAX_PREFIXES_PER_FILE = 50;
@@ -135,13 +143,11 @@ export class PathRouter {
             return new Map([[this.encodeIdentifier("remaining"), values]]);
         }
 
-        let ourSpec = authorityLookup.getOurSpec();
-
         let prefixes = ourSpec.prefixes.slice();
-        sort(prefixes, x => x.prefix.length);
+        sort(prefixes, x => x.length);
         function getPrefix(path: string): string | undefined {
             for (let prefix of prefixes) {
-                if (path.startsWith(prefix.prefix)) return prefix.prefix;
+                if (path.startsWith(prefix) && prefix !== path) return prefix;
             }
             return undefined;
         }
@@ -216,7 +222,9 @@ export class PathRouter {
                 let hashIndex = getPathDepth(prefix);
                 for (let value of values) {
                     let key = getPathIndex(value.path, hashIndex);
-                    if (key === undefined) throw new Error(`Impossible, hash index ${hashIndex} is out of range for path ${value.path}, but it matched the prefix ${prefix}`);
+                    if (key === undefined) {
+                        throw new Error(`Impossible, hash index ${hashIndex} is out of range for path ${value.path}, but it matched the prefix ${prefix}`);
+                    }
                     let route = this.getSingleKeyRoute(key);
                     let routeIndex = Math.floor(route * splitCount);
                     let routeValues = byRouteGroup.get(routeIndex);
@@ -253,7 +261,7 @@ export class PathRouter {
         // Match all remaining, as we don't store enough information to know what prefixes they excluded
         if (decodeObj === "remaining") return true;
 
-        let ourHashes = authority.prefixes.map(x => this.getPrefixHash(x.prefix));
+        let ourHashes = authority.prefixes.map(x => this.getPrefixHash(x));
         // If it pulled off some values as prefixes, but we did full path hashes, then the hashes are going to be totally different. And so there could easily be overlap.
         if (decodeObj.prefixHashes.some(x => !ourHashes.includes(x))) return true;
         // However, if we hash everything the same, then the overlap is purely a case of if the route ranges overlap. 
@@ -297,12 +305,12 @@ export class PathRouter {
 
         let groupByPrefixHash = new Map<string, AuthoritySpec[]>();
         for (let source of allSources) {
-            let usedPrefixes = source.authoritySpec.prefixes.map(x => x.prefix);
+            let usedPrefixes = source.authoritySpec.prefixes;
 
             if (target.excludeDefault) {
                 // This relaxes the restrictions, as with no default hashes, it means if a value isn't in one of our prefixes, even if it's inconsistent in the sources, it's fine. 
                 // (And... Otherwise, we can't filter these at all because otherwise the default/remaining group will be different, even if it's a superset/subset relationship)
-                let targetUsedPrefixes = new Set(target.prefixes.map(x => x.prefix));
+                let targetUsedPrefixes = new Set(target.prefixes);
                 usedPrefixes = usedPrefixes.filter(x => targetUsedPrefixes.has(x));
             }
             let prefixHash = sha256(JSON.stringify(usedPrefixes.sort()));
@@ -321,17 +329,17 @@ export class PathRouter {
         }
 
         function hashesSameAsTarget(spec: AuthoritySpec): boolean {
-            let targetUsedPrefixes = new Set(target.prefixes.map(x => x.prefix));
+            let targetUsedPrefixes = new Set(target.prefixes);
             if (target.excludeDefault) {
                 // If target is excluding default, then it doesn't matter if the spec is including it. We're only looking for matching values in target.
-                return spec.prefixes.every(x => targetUsedPrefixes.has(x.prefix));
+                return spec.prefixes.every(x => targetUsedPrefixes.has(x));
             }
             // if !targetExcludeDefault, then !spec.excludeDefault, because we filtered out other cases earlier (so we don't need to handle spec.
 
             if (spec.prefixes.length !== target.prefixes.length) return false;
             // Otherwise, All of the prefixes have to be identical
             for (let prefix of spec.prefixes) {
-                if (!targetUsedPrefixes.has(prefix.prefix)) return false;
+                if (!targetUsedPrefixes.has(prefix)) return false;
             }
             return true;
         }
@@ -432,7 +440,7 @@ export class PathRouter {
         let allSources = authorityLookup.getTopologySync();
         allSources = allSources.filter(x => !isOwnNodeId(x.nodeId));
         let nestedMatches = allSources.filter(x =>
-            x.authoritySpec.prefixes.some(y => path.startsWith(y.prefix) && y.prefix !== path)
+            x.authoritySpec.prefixes.some(y => path.startsWith(y) && y !== path)
             && this.matchesAuthoritySpec(x.authoritySpec, path)
         );
         if (nestedMatches.length > 0) {
@@ -451,10 +459,7 @@ export class PathRouter {
         let fullSources = this.getAuthoritySources({
             target: {
                 nodeId: "",
-                prefixes: [{
-                    prefix: path,
-                    hashIndex: getPathDepth(path),
-                }],
+                prefixes: [path],
                 routeStart: 0,
                 routeEnd: 1,
                 excludeDefault: true,

package/src/0-path-value-core/PathRouterRouteOverride.ts

@@ -62,7 +62,7 @@ export const hasPrefixHash = cacheLimited(1000 * 10,
     (config: { spec: AuthoritySpec, prefixHash: string }) => {
         let { spec, prefixHash } = config;
         for (let prefix of spec.prefixes) {
-            if (getPrefixHash(prefix.prefix) === prefixHash) {
+            if (getPrefixHash(prefix) === prefixHash) {
                 return true;
             }
         }

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -16,7 +16,12 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
 
     if (!range) {
         if (defaultToAll) {
-            return getAllAuthoritySpec();
+            return {
+                nodeId: "",
+                routeStart: 0,
+                routeEnd: 1,
+                prefixes: prefixes,
+            };
         }
         return undefined;
     }
@@ -38,10 +43,7 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
         nodeId: "",
         routeStart: rangeStart,
         routeEnd: rangeEnd,
-        prefixes: usePrefixes.map(prefix => ({
-            prefix,
-            hashIndex: getPathDepth(prefix),
-        })),
+        prefixes: usePrefixes,
         excludeDefault: excludeDefault || undefined,
     };
 }

package/src/0-path-value-core/hackedPackedPathParentFiltering.ts

@@ -20,7 +20,7 @@ let isPrefix = cache((path: string) => {
     let spec = authorityLookup.getOurSpec();
     if (!spec) return false;
     for (let prefix of spec.prefixes) {
-        if (path.startsWith(prefix.prefix)) return true;
+        if (path.startsWith(prefix)) return true;
     }
     return false;
 });

package/src/0-path-value-core/pathValueArchives.ts

@@ -13,6 +13,7 @@ import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
 import { devDebugbreak, isNoNetwork } from "../config";
 import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
 import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { authorityLookup } from "./AuthorityLookup";
 
 export const archives = lazy(() => wrapArchivesWithCache(getArchives("path-values/")));
 export const archivesLocks = lazy(() => getArchives("path-values-locks/"));
@@ -161,7 +162,7 @@ export class PathValueArchives {
 
     @measureFnc
     public async archiveValues(values: PathValue[]) {
-        let parts = PathRouter.getSelfPathIdentifierTargets(values);
+        let parts = PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
 
         for (let [pathIdentifier, values] of parts) {
             let encodedObj = await this.encodeValuePaths(values);

package/src/2-proxy/archiveMoveHarness.ts

@@ -7,7 +7,7 @@ import { PathValue, VALUE_GC_THRESHOLD, FILE_VALUE_COUNT_LIMIT, FILE_SIZE_LIMIT,
 import { getSingleSizeEstimate } from "../5-diagnostics/shared";
 import { logNodeStats } from "../-0-hooks/hooks";
 import debugbreak from "debugbreak";
-import { AuthorityEntry } from "../0-path-value-core/AuthorityLookup";
+import { AuthorityEntry, authorityLookup } from "../0-path-value-core/AuthorityLookup";
 import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
 
@@ -249,7 +249,7 @@ export async function runArchiveMover(config: {
 
 
             for (let [key, values] of Object.entries(result.newValues)) {
-                let targets = PathRouter.getSelfPathIdentifierTargets(values);
+                let targets = PathRouter.getPathIdentifierTargets(values, authority);
                 for (let [target, values] of targets) {
                     await addValues(target + "/", values, key);
                 }

package/src/2-proxy/garbageCollection.ts

@@ -19,6 +19,7 @@ import { getStorageDir, getSubFolder } from "../fs";
 import fs from "fs";
 import { sha256 } from "js-sha256";
 import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 
 
 export interface AliveChecker<T = unknown> {
@@ -34,7 +35,7 @@ export interface AliveChecker<T = unknown> {
      *  NOTE: The value might only have a single heavily nested value, not being a valid object. If the object
      *      lookups broken, probably just return false, to clean up the straggling values.
      */
-    isAlive: (value: T, path: string[]) => boolean | unknown;
+    isAlive: (value: T, path: string[], valuePath: PathValue | undefined) => boolean | unknown;
 
     /** Alias for lockBasedDelete & unsafeLiveReads */
     lock?: boolean;
@@ -254,7 +255,7 @@ function sandboxedIsAlive(config: {
             proxy = proxy[part];
         }
         try {
-            return !!atomicRaw(checker.isAlive(proxy, path));
+            return !!atomicRaw(checker.isAlive(proxy, path, undefined));
         } catch {
             config.errors.count++;
             return true;
@@ -308,7 +309,7 @@ function sandboxedGetLocks(config: {
 
     let evalResult = getWatchEvaluator()(() => {
         let value = proxyWatcher.getCallbackPathValue(pathStr);
-        return !!atomic(checker.isAlive(value, path));
+        return !!atomic(checker.isAlive(pathValueSerializer.getPathValue(value), path, value));
     }, { getReadLocks: true });
 
     return evalResult.readLocks!;

package/src/3-path-functions/syncSchema.ts

@@ -2,7 +2,7 @@ import fs from "fs";
 import { cache, lazy } from "socket-function/src/caching";
 import { getStringKeys, sort } from "socket-function/src/misc";
 import { Args } from "socket-function/src/types";
-import { appendToPathStr, getPathFromStr, getPathStr, rootPathStr } from "../path";
+import { appendToPathStr, getPathFromStr, getPathStr, joinPathStres, rootPathStr } from "../path";
 import { writeFunctionCall } from "./PathFunctionHelpers";
 import { CallSpec, functionSchema } from "./PathFunctionRunner";
 import { getDomain, isLocal } from "../config";
@@ -323,6 +323,7 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
                 createObjectAliveChecker(data, path, gcDelay);
             }
 
+
             let undoChecks = Schema2Fncs.getAllUndoChecks(schema);
             for (let checkObj of undoChecks) {
                 // Need at least one check
@@ -368,7 +369,10 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
             }
 
             // Register all root lookups as prefixes.
-            let paths = Schema2Fncs.getLookups(schema).map(x => getPathStr(x));
+            let dataRoot = getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Data);
+            let paths = Schema2Fncs.getLookups(schema).map(x =>
+                joinPathStres(dataRoot, getPathStr(x))
+            );
             // NOTE: This will break with wildcards, but... we also take only the highest level lookups, so it shouldn't be an issue
             sort(paths, x => x.length);
             let rootLookups = new Set<string>();

package/src/4-deploy/edgeClientWatcher.tsx

@@ -16,6 +16,11 @@ import { lazy } from "socket-function/src/caching";
 import { Button } from "../library-components/Button";
 import { updateRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
 
+setImmediate(() => {
+    // Modal
+    import("../5-diagnostics/Modal");
+});
+
 const SWITCH_SERVER_TIMEOUT = timeInSecond * 15;
 const MAX_DISPLAY_INTERVAL = timeInMinute * 5;
 const FINAL_DELAY = timeInSecond * 30;

package/src/archiveapps/archiveJoinEntry.ts

@@ -2,23 +2,29 @@ import "../inject";
 
 import { logErrors } from "../errors";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
-import { FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
+import { PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 import { ArchiveTransaction, FileInfo } from "../0-path-value-core/archiveLocks/ArchiveLocks";
 import { formatNumber } from "socket-function/src/formatting/format";
-import { sort } from "socket-function/src/misc";
+import { isNodeTrue, sort } from "socket-function/src/misc";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { magenta } from "socket-function/src/formatting/logColors";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
+import yargs from "yargs";
 
-async function runGenesisJoinIteration() {
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "If true, we only join genesis values, but we join in a loop, otherwise we just join absolutely everything, and then exit." })
+    .argv || {}
+    ;
+
+
+async function runGenesisJoinIteration(config?: { force?: boolean }) {
     let authoritySpec = await getOurAuthoritySpec(true);
     let locker = await pathValueArchives.getArchiveLocker();
-    let maxAge = Date.now() - VALUE_GC_THRESHOLD;
 
     let readCache = new Map<string, Buffer>();
     let reread = true;
@@ -31,15 +37,16 @@ async function runGenesisJoinIteration() {
             // Merge the newest first, so if we hit a big file, we can just ignore it,
             //  and next time merge the smaller files after it.
             sort(valueFiles, x => -x.createTime);
-            valueFiles = valueFiles.filter(x => {
-                let obj = pathValueArchives.decodeDataPath(x.file);
-                if (!obj.minTime) return false;
-                if (obj.sourceType !== "genesis") return false;
-                return x.createTime >= maxAge;
-            });
+            if (!config?.force) {
+                valueFiles = valueFiles.filter(x => {
+                    let obj = pathValueArchives.decodeDataPath(x.file);
+                    if (!obj.minTime) return false;
+                    return obj.sourceType === "genesis";
+                });
+            }
             let withinTimeRangeCount = valueFiles.length;
 
-            if (valueFiles.length < 3) return [];
+            if (valueFiles.length < 3 && !config?.force) return [];
 
             await measureBlock(async function locker_readFiles() {
                 let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
@@ -65,28 +72,20 @@ async function runGenesisJoinIteration() {
             let allValues: PathValue[][] = [];
             for (let valueFile of valueFiles) {
                 let buffer = readCache.get(valueFile.file)!;
-                if (totalSize + buffer.length > FILE_SIZE_LIMIT) {
-                    break;
-                }
                 let newValues = await PathValueArchives.loadValuesFromBuffer({
                     path: valueFile.file,
                     data: buffer,
                 });
 
                 let readValues = newValues.values;
-                if (readValues.length + totalCount > FILE_VALUE_COUNT_LIMIT) {
-                    break;
-                }
                 allValues.push(readValues);
                 usedFiles.push(valueFile);
                 totalSize += buffer.length;
                 totalCount += readValues.length;
             }
-            if (usedFiles.length <= 1) return [];
 
             let allCombinedValues = allValues.flat();
 
-
             let transaction: ArchiveTransaction = {
                 createFiles: [],
                 deleteFiles: [],
@@ -95,7 +94,7 @@ async function runGenesisJoinIteration() {
                 transaction.deleteFiles.push(file);
             }
 
-            let targets = PathRouter.getSelfPathIdentifierTargets(allCombinedValues);
+            let targets = PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
 
                 let dataObj = await pathValueArchives.encodeValuePaths(values, {
@@ -123,7 +122,17 @@ async function main() {
     disablePathAuditer();
     await Querysub.hostService("join");
 
-    await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
+    if (yargObj.watch) {
+        await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
+    } else {
+        try {
+            await runGenesisJoinIteration({ force: true });
+        } catch (error) {
+            console.error(error);
+        } finally {
+            process.exit();
+        }
+    }
 }
 
 main().catch(logErrors);

package/src/functional/promiseCache.ts

@@ -34,7 +34,10 @@ export function cacheAsyncLimited<Arg, Return>(limit: number, fnc: (arg: Arg) =>
 export const cacheAsyncSynced = cacheAsyncLimitedJSON;
 export function cacheAsyncLimitedJSON<Arg, Return>(
     limit: number,
-    fnc: (arg: Arg) => Promise<Return>
+    fnc: (arg: Arg) => Promise<Return>,
+    config?: {
+        notReadyValue?: Return;
+    }
 ): ((arg: Arg) => Return | undefined) & {
     clear(): void;
     promise: (arg: Arg) => Promise<Return>;
@@ -71,6 +74,6 @@ export function cacheAsyncLimitedJSON<Arg, Return>(
             }
         );
         proxyWatcher.triggerOnPromiseFinish(promise, { waitReason: fnc.toString() });
-        return undefined;
+        return config?.notReadyValue;
     };
 }

package/src/path.ts

@@ -181,8 +181,12 @@ export function removePathLastPart(pathStr: string) {
     return pathStr.slice(0, getStartOfLastPart(pathStr));
 }
 
+let lastPastPath = { path: "", depth: 0 };
 /** getPathDepth(pathStr) === getPathFromStr(pathStr).length */
 export function getPathDepth(pathStr: string): number {
+    if (lastPastPath.path === pathStr) {
+        return lastPastPath.depth;
+    }
     // Start at 1, to skip the first pathDelimit
     let index = 1;
     // Start at -1, as the last loop doesn't have pathDelimitEscaped, but is still incremented
@@ -191,6 +195,8 @@ export function getPathDepth(pathStr: string): number {
         count++;
         index = pathStr.indexOf(pathDelimitEscaped, index) + 1;
     }
+    lastPastPath.path = pathStr;
+    lastPastPath.depth = count;
     return count;
 }
 

package/tempnotes.txt

@@ -12,8 +12,6 @@ BUGS:
 
 
 
-4) deploy ON THE SITE, to deploy our prefixes
-    - Do it via the site, to commit, publish, etc.
 3) gc, to split it into multiple directories
 4) Verify the site still works
 

package/test.ts

@@ -3,6 +3,7 @@ chdir("D:/repos/qs-cyoa/");
 
 import "./inject";
 
+import path from "path";
 import { list, sort } from "socket-function/src/misc";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { PathValueArchives, archives, archivesLocks, archivesRecycleBin, pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
@@ -19,26 +20,29 @@ import { unique } from "./src/misc";
 import { getAllAuthoritySpec, getOurAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
+import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
 
 async function main() {
-    let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
-    paths = paths.filter(x => x.startsWith("P!REMAINING/"));
-    for (let path of paths) {
-        let data = await archives().get(path);
-        if (!data) {
-            console.log(`Missing data file: ${path}`);
-            continue;
-        }
-        let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
+    let prefixes = await getShardPrefixes();
+    console.log(prefixes);
+    // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
+    // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
+    // for (let path of paths) {
+    //     let data = await archives().get(path);
+    //     if (!data) {
+    //         console.log(`Missing data file: ${path}`);
+    //         continue;
+    //     }
+    //     let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
 
-        for (let value of loadObj.values) {
-            if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
-                throw new Error(`Value ${value.path} does not match authority spec`);
-            }
-            console.log(pathValueSerializer.getPathValue(value));
-        }
-    }
-    return;
+    //     for (let value of loadObj.values) {
+    //         if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
+    //             throw new Error(`Value ${value.path} does not match authority spec`);
+    //         }
+    //         console.log(pathValueSerializer.getPathValue(value));
+    //     }
+    // }
+    // return;
 
     // let path = "P!REMAINING/data_37c5d323e0f9a083.1ed7a7340a015680.querysubtest.com:52026_1775097754671.579_3_1_337_1775097652829.5793_1775097652829.5793_genesis_.data";
     // let value = await archives().get(path);
@@ -54,6 +58,7 @@ async function main() {
     //     return path.split("/").at(-1)!;
     // }
 
+    //*
     let allValuesObj = await pathValueArchives.loadValues({
         nodeId: "",
         prefixes: [],
@@ -62,6 +67,7 @@ async function main() {
     });
     let allValues = Object.values(allValuesObj.values).flat();
     console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
+    //*/
 
     // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
 

package/src/archiveapps/archiveMergeEntry.tsx

@@ -1,50 +0,0 @@
-import "../inject";
-
-import { Querysub } from "../4-querysub/QuerysubController";
-import { logErrors } from "../errors";
-import { timeInDay, timeInHour } from "socket-function/src/misc";
-import { runArchiveMover } from "../2-proxy/archiveMoveHarness";
-import { PathValue, compareTime } from "../0-path-value-core/pathValueCore";
-import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
-
-const MERGE_DELAY = timeInHour;
-
-async function mergeFiles() {
-    await runArchiveMover({
-        debugName: "merge",
-        outputType: "merge",
-        async runMover(config) {
-            let values = config.values;
-            let latestValues = new Map<string, PathValue>();
-            for (let value of values) {
-                let prev = latestValues.get(value.path);
-                if (prev && compareTime(value.time, prev.time) < 0) continue;
-                latestValues.set(value.path, value);
-            }
-
-            values = Array.from(latestValues.values());
-            values = values.filter(x => !x.canGCValue);
-
-            return {
-                newValues: {
-                    "": values,
-                },
-            };
-        },
-    });
-}
-
-async function main() {
-    disablePathAuditer();
-    await Querysub.hostService("merge");
-    await mergeFiles();
-    process.exit();
-
-    // while (true) {
-    //     await mergeFiles();
-    //     let delay = MERGE_DELAY * (1 + Math.random() * 0.1);
-    //     await new Promise(resolve => setTimeout(resolve, delay));
-    // }
-}
-
-main().catch(logErrors);

package/src/diagnostics/logs/BrowserLargeFileCache.ts

@@ -1,64 +0,0 @@
-import { SocketFunctionClientHook } from "socket-function/SocketFunctionTypes";
-import { cache, lazy } from "socket-function/src/caching";
-import { decodeCborx, encodeCborx } from "../../misc/cloneHelpers";
-import { sha256 } from "js-sha256";
-import { errorToUndefined } from "../../errors";
-import { isNode } from "typesafecss";
-
-let getRootDirectory = lazy(async () => {
-    await navigator.storage.persist();
-    return navigator.storage.getDirectory();
-});
-
-export const getBrowserLargeFileCache = cache((name: string) => new BrowserLargeFileCache(name));
-
-export class BrowserLargeFileCache {
-    constructor(private name: string) { }
-
-    private getDir = lazy(async () => {
-        let root = await getRootDirectory();
-        return await root.getDirectoryHandle(this.name, { create: true });
-    });
-
-    public async set(key: string, value: Buffer) {
-        let dir = await this.getDir();
-        let file = await dir.getFileHandle(key, { create: true });
-        let writable = await file.createWritable();
-        await writable.write(value);
-        await writable.close();
-    }
-    public async get(key: string): Promise<Buffer | undefined> {
-        let dir = await this.getDir();
-        try {
-            let file = await dir.getFileHandle(key, { create: false });
-            let readable = await file.getFile();
-            return Buffer.from(await readable.arrayBuffer());
-        } catch {
-            return undefined;
-        }
-    }
-}
-
-/** Cache key = [args, functionName, classGuid]
- *      - Not nodeId, as that change so frequently, so caching based on server is not usually useful.
- *          - If you want to cache based on nodeId, pass it as an unused arg.
- * 
- *  Uses a file per key, and never cleans them up. So... basically, this sucks, but, it should
- *      work for a few specific functions with large and consistent values.
- */
-export const cacheCalls: SocketFunctionClientHook = async config => {
-    if (isNode()) return;
-
-    let { args, functionName, classGuid } = config.call;
-    let bucket = sha256(JSON.stringify({ functionName, classGuid }));
-    let cache = getBrowserLargeFileCache(bucket);
-    let key = sha256(encodeCborx(args));
-    let cachedValue = await cache.get(key);
-    if (cachedValue) {
-        config.overrideResult = decodeCborx(cachedValue);
-        return;
-    }
-    config.onResult.push(async (result) => {
-        await errorToUndefined(cache.set(key, encodeCborx(result)));
-    });
-};