querysub 0.404.0 → 0.405.0

package/bin/join.js

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
 
 require("typenode");
-require("../src/archiveapps/archiveGCEntry");
+require("../src/archiveapps/archiveJoinEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.404.0",
+  "version": "0.405.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathRouter.ts

@@ -1,5 +1,5 @@
 import { sort } from "socket-function/src/misc";
-import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1 } from "../path";
+import { getLastPathPart, getPathDepth, getPathIndex, getPathStr1, hack_stripPackedPath } from "../path";
 import { AuthorityEntry, authorityLookup } from "./AuthorityLookup";
 import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
@@ -59,8 +59,9 @@ export class PathRouter {
         path: string;
         spec: AuthoritySpec;
     }): number {
-        // NOTE: getSelfPathIdentifierTargets also hardcodes this logic, so it can hash many values quickly
+        // NOTE: getPathIdentifierTargets also hardcodes this logic, so it can hash many values quickly
         let { path, spec } = config;
+        path = hack_stripPackedPath(path);
         let override = getRoutingOverride(path);
         if (override) {
             if (!hasPrefixHash({ spec, prefixHash: override.prefixHash })) return -1;
@@ -70,7 +71,11 @@ export class PathRouter {
         let prefix = spec.prefixes.find(x => path.startsWith(x.prefix) && x.prefix !== path);
         if (prefix) {
             let key = getPathIndex(path, prefix.hashIndex);
-            if (key === undefined) throw new Error(`Impossible, hash index ${prefix.hashIndex} is out of range for path ${path}, but it matched the prefix ${prefix.prefix}`);
+            if (key === undefined) {
+                require("debugbreak")(2);
+                debugger;
+                throw new Error(`Impossible, hash index ${prefix.hashIndex} is out of range for path ${path}, but it matched the prefix ${prefix.prefix}`);
+            }
             return this.getSingleKeyRoute(key);
         }
         if (spec.excludeDefault) return -1;
@@ -107,13 +112,13 @@ export class PathRouter {
     private static encodeIdentifier(config: { prefixes: string[]; rangeStart: number; rangeEnd: number } | "remaining"): string {
         if (config === "remaining") return "P!REMAINING";
         let { prefixes, rangeStart, rangeEnd } = config;
-        return ["P", rangeStart.toString(), rangeEnd.toString(), ...prefixes.map(x => this.getPrefixHash(x))].join("!");
+        return ["P", rangeStart.toString(), rangeEnd.toString(), ...prefixes.map(x => this.getPrefixHash(x)), ""].join("!");
 
     }
     private static decodeIdentifier(identifier: string): { prefixHashes: string[]; rangeStart: number; rangeEnd: number } | "remaining" {
         if (!identifier.startsWith("P")) return "remaining";
         let parts = identifier.split("!");
-        if (parts[1] === "REMAINING") return "remaining";
+        if (parts[1].startsWith("REMAINING")) return "remaining";
         return {
             rangeStart: parseFloat(parts[1]),
             rangeEnd: parseFloat(parts[2]),
@@ -125,7 +130,7 @@ export class PathRouter {
     // NOTE: Encodes all the data, even if it matches our spec or not. If you don't want this, you have to filter first
     //  - If this becomes an issue we COULD filter, as we can do it quickly, but I don't think it is required, as all the present usecases prefilter anyways.
     @measureFnc
-    public static getSelfPathIdentifierTargets(values: PathValue[]): Map<string, PathValue[]> {
+    public static getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Map<string, PathValue[]> {
         // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
         const MAX_PREFIXES_PER_FILE = 50;
@@ -135,13 +140,11 @@ export class PathRouter {
             return new Map([[this.encodeIdentifier("remaining"), values]]);
         }
 
-        let ourSpec = authorityLookup.getOurSpec();
-
         let prefixes = ourSpec.prefixes.slice();
         sort(prefixes, x => x.prefix.length);
         function getPrefix(path: string): string | undefined {
             for (let prefix of prefixes) {
-                if (path.startsWith(prefix.prefix)) return prefix.prefix;
+                if (path.startsWith(prefix.prefix) && prefix.prefix !== path) return prefix.prefix;
             }
             return undefined;
         }
@@ -165,6 +168,9 @@ export class PathRouter {
         }));
         sort(prefixGroups, x => -x.values.length);
 
+        require("debugbreak")(2);
+        debugger;
+
         let groups: {
             prefixes: string[];
             values: PathValue[][];
@@ -216,7 +222,9 @@ export class PathRouter {
                 let hashIndex = getPathDepth(prefix);
                 for (let value of values) {
                     let key = getPathIndex(value.path, hashIndex);
-                    if (key === undefined) throw new Error(`Impossible, hash index ${hashIndex} is out of range for path ${value.path}, but it matched the prefix ${prefix}`);
+                    if (key === undefined) {
+                        throw new Error(`Impossible, hash index ${hashIndex} is out of range for path ${value.path}, but it matched the prefix ${prefix}`);
+                    }
                     let route = this.getSingleKeyRoute(key);
                     let routeIndex = Math.floor(route * splitCount);
                     let routeValues = byRouteGroup.get(routeIndex);

package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx

@@ -16,7 +16,15 @@ export async function getOurAuthoritySpec(defaultToAll?: boolean): Promise<Autho
 
     if (!range) {
         if (defaultToAll) {
-            return getAllAuthoritySpec();
+            return {
+                nodeId: "",
+                routeStart: 0,
+                routeEnd: 1,
+                prefixes: prefixes.map(prefix => ({
+                    prefix,
+                    hashIndex: getPathDepth(prefix),
+                })),
+            };
         }
         return undefined;
     }

package/src/0-path-value-core/pathValueArchives.ts

@@ -13,6 +13,7 @@ import { createArchiveLocker2 } from "./archiveLocks/ArchiveLocks2";
 import { devDebugbreak, isNoNetwork } from "../config";
 import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
 import { AuthoritySpec, PathRouter } from "./PathRouter";
+import { authorityLookup } from "./AuthorityLookup";
 
 export const archives = lazy(() => wrapArchivesWithCache(getArchives("path-values/")));
 export const archivesLocks = lazy(() => getArchives("path-values-locks/"));
@@ -161,7 +162,7 @@ export class PathValueArchives {
 
     @measureFnc
     public async archiveValues(values: PathValue[]) {
-        let parts = PathRouter.getSelfPathIdentifierTargets(values);
+        let parts = PathRouter.getPathIdentifierTargets(values, authorityLookup.getOurSpec());
 
         for (let [pathIdentifier, values] of parts) {
             let encodedObj = await this.encodeValuePaths(values);

package/src/2-proxy/archiveMoveHarness.ts

@@ -7,7 +7,7 @@ import { PathValue, VALUE_GC_THRESHOLD, FILE_VALUE_COUNT_LIMIT, FILE_SIZE_LIMIT,
 import { getSingleSizeEstimate } from "../5-diagnostics/shared";
 import { logNodeStats } from "../-0-hooks/hooks";
 import debugbreak from "debugbreak";
-import { AuthorityEntry } from "../0-path-value-core/AuthorityLookup";
+import { AuthorityEntry, authorityLookup } from "../0-path-value-core/AuthorityLookup";
 import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
 
@@ -249,7 +249,7 @@ export async function runArchiveMover(config: {
 
 
             for (let [key, values] of Object.entries(result.newValues)) {
-                let targets = PathRouter.getSelfPathIdentifierTargets(values);
+                let targets = PathRouter.getPathIdentifierTargets(values, authority);
                 for (let [target, values] of targets) {
                     await addValues(target + "/", values, key);
                 }

package/src/3-path-functions/syncSchema.ts

@@ -2,7 +2,7 @@ import fs from "fs";
 import { cache, lazy } from "socket-function/src/caching";
 import { getStringKeys, sort } from "socket-function/src/misc";
 import { Args } from "socket-function/src/types";
-import { appendToPathStr, getPathFromStr, getPathStr, rootPathStr } from "../path";
+import { appendToPathStr, getPathFromStr, getPathStr, joinPathStres, rootPathStr } from "../path";
 import { writeFunctionCall } from "./PathFunctionHelpers";
 import { CallSpec, functionSchema } from "./PathFunctionRunner";
 import { getDomain, isLocal } from "../config";
@@ -323,6 +323,7 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
                 createObjectAliveChecker(data, path, gcDelay);
             }
 
+
             let undoChecks = Schema2Fncs.getAllUndoChecks(schema);
             for (let checkObj of undoChecks) {
                 // Need at least one check
@@ -368,7 +369,10 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
             }
 
             // Register all root lookups as prefixes.
-            let paths = Schema2Fncs.getLookups(schema).map(x => getPathStr(x));
+            let dataRoot = getProxyPath(() => functionSchema()[getDomain()].PathFunctionRunner[moduleId].Data);
+            let paths = Schema2Fncs.getLookups(schema).map(x =>
+                joinPathStres(dataRoot, getPathStr(x))
+            );
             // NOTE: This will break with wildcards, but... we also take only the highest level lookups, so it shouldn't be an issue
             sort(paths, x => x.length);
             let rootLookups = new Set<string>();

package/src/4-deploy/edgeClientWatcher.tsx

@@ -16,6 +16,11 @@ import { lazy } from "socket-function/src/caching";
 import { Button } from "../library-components/Button";
 import { updateRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
 
+setImmediate(() => {
+    // Modal
+    import("../5-diagnostics/Modal");
+});
+
 const SWITCH_SERVER_TIMEOUT = timeInSecond * 15;
 const MAX_DISPLAY_INTERVAL = timeInMinute * 5;
 const FINAL_DELAY = timeInSecond * 30;

package/src/archiveapps/archiveJoinEntry.ts

@@ -2,23 +2,29 @@ import "../inject";
 
 import { logErrors } from "../errors";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
-import { FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
+import { PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 import { ArchiveTransaction, FileInfo } from "../0-path-value-core/archiveLocks/ArchiveLocks";
 import { formatNumber } from "socket-function/src/formatting/format";
-import { sort } from "socket-function/src/misc";
+import { isNodeTrue, sort } from "socket-function/src/misc";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { magenta } from "socket-function/src/formatting/logColors";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
+import yargs from "yargs";
 
-async function runGenesisJoinIteration() {
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "If true, we only join genesis values, but we join in a loop, otherwise we just join absolutely everything, and then exit." })
+    .argv || {}
+    ;
+
+
+async function runGenesisJoinIteration(config?: { force?: boolean }) {
     let authoritySpec = await getOurAuthoritySpec(true);
     let locker = await pathValueArchives.getArchiveLocker();
-    let maxAge = Date.now() - VALUE_GC_THRESHOLD;
 
     let readCache = new Map<string, Buffer>();
     let reread = true;
@@ -31,15 +37,16 @@ async function runGenesisJoinIteration() {
             // Merge the newest first, so if we hit a big file, we can just ignore it,
             //  and next time merge the smaller files after it.
             sort(valueFiles, x => -x.createTime);
-            valueFiles = valueFiles.filter(x => {
-                let obj = pathValueArchives.decodeDataPath(x.file);
-                if (!obj.minTime) return false;
-                if (obj.sourceType !== "genesis") return false;
-                return x.createTime >= maxAge;
-            });
+            if (!config?.force) {
+                valueFiles = valueFiles.filter(x => {
+                    let obj = pathValueArchives.decodeDataPath(x.file);
+                    if (!obj.minTime) return false;
+                    return obj.sourceType === "genesis";
+                });
+            }
             let withinTimeRangeCount = valueFiles.length;
 
-            if (valueFiles.length < 3) return [];
+            if (valueFiles.length < 3 && !config?.force) return [];
 
             await measureBlock(async function locker_readFiles() {
                 let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
@@ -65,28 +72,20 @@ async function runGenesisJoinIteration() {
             let allValues: PathValue[][] = [];
             for (let valueFile of valueFiles) {
                 let buffer = readCache.get(valueFile.file)!;
-                if (totalSize + buffer.length > FILE_SIZE_LIMIT) {
-                    break;
-                }
                 let newValues = await PathValueArchives.loadValuesFromBuffer({
                     path: valueFile.file,
                     data: buffer,
                 });
 
                 let readValues = newValues.values;
-                if (readValues.length + totalCount > FILE_VALUE_COUNT_LIMIT) {
-                    break;
-                }
                 allValues.push(readValues);
                 usedFiles.push(valueFile);
                 totalSize += buffer.length;
                 totalCount += readValues.length;
             }
-            if (usedFiles.length <= 1) return [];
 
             let allCombinedValues = allValues.flat();
 
-
             let transaction: ArchiveTransaction = {
                 createFiles: [],
                 deleteFiles: [],
@@ -95,7 +94,7 @@ async function runGenesisJoinIteration() {
                 transaction.deleteFiles.push(file);
             }
 
-            let targets = PathRouter.getSelfPathIdentifierTargets(allCombinedValues);
+            let targets = PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
 
                 let dataObj = await pathValueArchives.encodeValuePaths(values, {
@@ -123,7 +122,17 @@ async function main() {
     disablePathAuditer();
     await Querysub.hostService("join");
 
-    await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
+    if (yargObj.watch) {
+        await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
+    } else {
+        try {
+            await runGenesisJoinIteration({ force: true });
+        } catch (error) {
+            console.error(error);
+        } finally {
+            process.exit();
+        }
+    }
 }
 
 main().catch(logErrors);

package/tempnotes.txt

@@ -12,8 +12,6 @@ BUGS:
 
 
 
-4) deploy ON THE SITE, to deploy our prefixes
-    - Do it via the site, to commit, publish, etc.
 3) gc, to split it into multiple directories
 4) Verify the site still works
 

package/test.ts

@@ -3,6 +3,7 @@ chdir("D:/repos/qs-cyoa/");
 
 import "./inject";
 
+import path from "path";
 import { list, sort } from "socket-function/src/misc";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { PathValueArchives, archives, archivesLocks, archivesRecycleBin, pathValueArchives } from "./src/0-path-value-core/pathValueArchives";
@@ -19,26 +20,31 @@ import { unique } from "./src/misc";
 import { getAllAuthoritySpec, getOurAuthoritySpec } from "./src/0-path-value-core/PathRouterServerAuthoritySpec";
 import { PathRouter } from "./src/0-path-value-core/PathRouter";
 import { pathValueSerializer } from "./src/-h-path-value-serialize/PathValueSerializer";
+import { getShardPrefixes } from "./src/0-path-value-core/ShardPrefixes";
 
 async function main() {
-    let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
-    paths = paths.filter(x => x.startsWith("P!REMAINING/"));
-    for (let path of paths) {
-        let data = await archives().get(path);
-        if (!data) {
-            console.log(`Missing data file: ${path}`);
-            continue;
-        }
-        let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
+    let prefixes = await getShardPrefixes();
+    console.log(prefixes);
+    let deployPath = path.resolve("./deploy.ts");
+    await import(deployPath);
+    // let paths = await pathValueArchives.getValuePaths(getAllAuthoritySpec());
+    // paths = paths.filter(x => x.startsWith("P!REMAINING/"));
+    // for (let path of paths) {
+    //     let data = await archives().get(path);
+    //     if (!data) {
+    //         console.log(`Missing data file: ${path}`);
+    //         continue;
+    //     }
+    //     let loadObj = await PathValueArchives.loadValuesFromBuffer({ path, data });
 
-        for (let value of loadObj.values) {
-            if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
-                throw new Error(`Value ${value.path} does not match authority spec`);
-            }
-            console.log(pathValueSerializer.getPathValue(value));
-        }
-    }
-    return;
+    //     for (let value of loadObj.values) {
+    //         if (!PathRouter.matchesAuthoritySpec(getAllAuthoritySpec(), value.path)) {
+    //             throw new Error(`Value ${value.path} does not match authority spec`);
+    //         }
+    //         console.log(pathValueSerializer.getPathValue(value));
+    //     }
+    // }
+    // return;
 
     // let path = "P!REMAINING/data_37c5d323e0f9a083.1ed7a7340a015680.querysubtest.com:52026_1775097754671.579_3_1_337_1775097652829.5793_1775097652829.5793_genesis_.data";
     // let value = await archives().get(path);
@@ -54,6 +60,7 @@ async function main() {
     //     return path.split("/").at(-1)!;
     // }
 
+    /*
     let allValuesObj = await pathValueArchives.loadValues({
         nodeId: "",
         prefixes: [],
@@ -62,6 +69,7 @@ async function main() {
     });
     let allValues = Object.values(allValuesObj.values).flat();
     console.log(`Total values loaded: ${formatNumber(allValues.length)}`);
+    */
 
     // const badDeleteTransaction = "transaction_   tSeqNum=887   tWriteTime=1774715868374.9143   thread=9dc0   cCount=14   dCount=3482   create=X,X,X,X,X,X,X,X,X,X,X,X,X,X   delete=2@0977,3@0977,4@0977,1@0a2a,2@0a2a,3@0a2a,4@0a2...   .transaction";
 

package/src/archiveapps/archiveMergeEntry.tsx

@@ -1,50 +0,0 @@
-import "../inject";
-
-import { Querysub } from "../4-querysub/QuerysubController";
-import { logErrors } from "../errors";
-import { timeInDay, timeInHour } from "socket-function/src/misc";
-import { runArchiveMover } from "../2-proxy/archiveMoveHarness";
-import { PathValue, compareTime } from "../0-path-value-core/pathValueCore";
-import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
-
-const MERGE_DELAY = timeInHour;
-
-async function mergeFiles() {
-    await runArchiveMover({
-        debugName: "merge",
-        outputType: "merge",
-        async runMover(config) {
-            let values = config.values;
-            let latestValues = new Map<string, PathValue>();
-            for (let value of values) {
-                let prev = latestValues.get(value.path);
-                if (prev && compareTime(value.time, prev.time) < 0) continue;
-                latestValues.set(value.path, value);
-            }
-
-            values = Array.from(latestValues.values());
-            values = values.filter(x => !x.canGCValue);
-
-            return {
-                newValues: {
-                    "": values,
-                },
-            };
-        },
-    });
-}
-
-async function main() {
-    disablePathAuditer();
-    await Querysub.hostService("merge");
-    await mergeFiles();
-    process.exit();
-
-    // while (true) {
-    //     await mergeFiles();
-    //     let delay = MERGE_DELAY * (1 + Math.random() * 0.1);
-    //     await new Promise(resolve => setTimeout(resolve, delay));
-    // }
-}
-
-main().catch(logErrors);