querysub 0.403.0 → 0.404.0

package/src/4-querysub/querysubPrediction.ts

@@ -1,34 +1,38 @@
-import { cacheJSONArgsEqual, cacheWeak, lazy } from "socket-function/src/caching";
-import { pathValueCommitter } from "../0-path-value-core/PathValueController";
-import { CLIENTSIDE_PREDICT_LEEWAY, MAX_ACCEPTED_CHANGE_AGE, MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, compareTime, debugTime, getCreatorId, getNextTime, lockToCallback, predictionLockVersion, timeMinusEpsilon } from "../0-path-value-core/pathValueCore";
-import { remoteWatcher } from "../1-path-client/RemoteWatcher";
-import { proxyWatcher, atomicObjectRead, DryRunResult, getCurrentCallCreationProxy, debug_getQueueOrder } from "../2-proxy/PathValueProxyWatcher";
-import { getPathFromProxy, getProxyPath } from "../2-proxy/pathValueProxy";
-import { CallSpec, FunctionResult, FunctionSpec, debugCallSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
+import { cacheJSONArgsEqual, lazy } from "socket-function/src/caching";
+import { MAX_CHANGE_AGE, PathValue, ReadLock, Time, authorityStorage, predictionLockVersion } from "../0-path-value-core/pathValueCore";
+import { validStateComputer } from "../0-path-value-core/ValidStateComputer";
+import { proxyWatcher, atomicObjectRead, DryRunResult, getCurrentCallCreationProxy } from "../2-proxy/PathValueProxyWatcher";
+import { getProxyPath } from "../2-proxy/pathValueProxy";
+import { CallSpec, FunctionResult, FunctionSpec, functionSchema, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
 import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathFunctionLoader";
 import { logErrors } from "../errors";
-import { getParentPathStr, getPathFromStr, getPathStr1, getPathStr2, getPathStr3 } from "../path";
-import { Querysub, QuerysubController, QuerysubControllerBase, baseAddCall, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
+import { getPathFromStr, getPathStr1 } from "../path";
+import { Querysub, QuerysubController, QuerysubControllerBase, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
 import { Benchmark } from "../diagnostics/benchmark";
 import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
-import { runInSerial } from "socket-function/src/batching";
-import { green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { magenta, red } from "socket-function/src/formatting/logColors";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
 
 import { setFlag } from "socket-function/require/compileFlags";
 import cbor from "cbor-x";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
-import { PromiseObj, isNode, nextId, sort } from "socket-function/src/misc";
-import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
-import { isLocal } from "../config";
-import { onAllPredictionsFinished } from "../-0-hooks/hooks";
-import { t } from "../2-proxy/schema2";
+import { PromiseObj, isNode } from "socket-function/src/misc";
+import { isPublic } from "../config";
 import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
 import { addEpsilons } from "../bits";
 setFlag(require, "cbor-x", "allowclient", true);
 const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
 
+let onPredictionFinishedCallbacks: Array<(data: { callId: string; result: FunctionResult; functionId: string }) => void> = [];
+
+export function onPredictionFinished(callback: (data: {
+    callId: string;
+    result: FunctionResult;
+    functionId: string;
+}) => void) {
+    onPredictionFinishedCallbacks.push(callback);
+}
 
 // NOTE: Most functions won't use this, as they should use regular api calls. However,
 //  as we are using paths for RequireJS, we explicitly need our local (when serverside).
@@ -120,8 +124,11 @@ function predictCallBase(config: {
     let functionResult: FunctionResult = {
         // Special prediction values (we don't even really need to put anything here, but it
         //  might as well have the right format, in case a bug causes it to be used somewhere)
-        runCount: -1,
+        lastInternalLoopCount: -1,
+        outerLoopCount: -1,
+        totalInternalLoopCount: -1,
         timeTaken: -1,
+        totalTime: -1,
         evalTime: 0,
     };
 
@@ -143,7 +150,6 @@ function predictCallBase(config: {
                 path: pathResultWrite,
                 // Technically we are depending on a future time, so... we are depending on ourself? And our value isn't undefined.
                 readIsTransparent: false,
-                keepRejectedUntil: call.runAtTime.time + CLIENTSIDE_PREDICT_LEEWAY
             }
         ],
         lockCount: 1,
@@ -169,9 +175,6 @@ function predictCallBase(config: {
             console.log(magenta(`Start predict call`), `${call.DomainName}.${call.FunctionId}`);
         }
 
-        // Watch the result, so we know when our prediction is rejected (which will be as soon as the result
-        //  has a real value).
-        remoteWatcher.watchLatest({ paths: [pathResultWrite], parentPaths: [] });
         let debugName = `[predict]|${call.DomainName}.${call.FunctionId}`;
 
         let dryRunResult: {
@@ -182,8 +185,13 @@ function predictCallBase(config: {
 
         let actualValueFinished = new PromiseObj();
         function onActualFinished() {
-            if (authorityStorage.getValueAtTime(pathResultWrite, undefined)?.value) {
-                actualValueFinished.resolve();
+            let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
+            let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
+            if (!result) return;
+            if (result.lastInternalLoopCount === -1) return;
+            actualValueFinished.resolve();
+            for (let callback of onPredictionFinishedCallbacks) {
+                callback({ callId: call.CallId, result, functionId: call.FunctionId });
             }
         }
         clientWatcher.setWatches({
@@ -191,40 +199,39 @@ function predictCallBase(config: {
             paths: new Set([pathResultWrite]),
             parentPaths: new Set(),
         });
-        try {
 
-            try {
-                let tempDryRunResult = await Promise.race([
-                    await getCallWrites({ call, debugName, overrides: config.overrides, useFinishReordering: true, metadata: config.metadata }),
-                    actualValueFinished.promise,
-                ]);
-                if (!tempDryRunResult) {
-                    if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
-                        console.log(magenta(`Abort predict call before prediction finished, already received call result`), `${call.DomainName}.${call.FunctionId}`);
-                    }
-                    return undefined;
-                }
-                dryRunResult = tempDryRunResult;
-            } catch (e: any) {
-
-                if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
-                    console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
-                } else {
-                    // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
-                    //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
-                    //  the race to predict the function against the server updating it, it is likely our prediction will now
-                    //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
-                    //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
+        setTimeout(() => {
+            clientWatcher.unwatch(onActualFinished);
+        }, 30 * 1000);
+
+        try {
+            let tempDryRunResult = await Promise.race([
+                await getCallWrites({ call, debugName, overrides: config.overrides, useFinishReordering: true, metadata: config.metadata }),
+                actualValueFinished.promise,
+            ]);
+            if (!tempDryRunResult) {
+                if (Querysub.DEBUG_PREDICTIONS || Querysub.DEBUG_CALLS) {
+                    console.log(magenta(`Abort predict call before prediction finished, already received call result`), `${call.DomainName}.${call.FunctionId}`);
                 }
-                return {
-                    writes: [],
-                    readPaths: new Set(),
-                    readParentPaths: new Set(),
-                    replacedWriteValues: [],
-                };
+                return undefined;
             }
-        } finally {
-            clientWatcher.unwatch(onActualFinished);
+            dryRunResult = tempDryRunResult;
+        } catch (e: any) {
+            if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
+                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
+            } else {
+                // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
+                //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
+                //  the race to predict the function against the server updating it, it is likely our prediction will now
+                //  be running before the latest write. In which case (as we don't really store write history), we will read undefined.
+                //  This isn't accurate, but... our write WILL almost certainly be wrong (as the value changed), so we don't log here.
+            }
+            return {
+                writes: [],
+                readPaths: new Set(),
+                readParentPaths: new Set(),
+                replacedWriteValues: [],
+            };
         }
         predictions = {
             writes: dryRunResult.writes,
@@ -274,14 +281,11 @@ function predictCallBase(config: {
             predict.lockCount = predictLocks.length;
         }
 
-        pathValueCommitter.ingestValues(predictions.writes, undefined, undefined);
-
-        // NOTE: We have to forcefully setup the lock watches, as we are no the authority on these paths
-        //  so it is odd for us to do this! It works out though, because the path is unique enough
-        //  that we don't have to worry about any other valid states coming in for our lock.
-        //  - Also, once we set up this lock, we should get full reject cascade support.
-        lockToCallback.watchLock(predictResultWrite.locks[0], [predictResultWrite]);
-        lockToCallback.watchLock(predictLocks[0], predictions.writes);
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: predictions.writes,
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+        });
 
         if (Querysub.DEBUG_PREDICTIONS) {
             console.log(magenta(`Finished and applied predict call`), `${call.DomainName}.${call.FunctionId}`);
@@ -295,13 +299,19 @@ function predictCallBase(config: {
         if (didCancel) return;
         didCancel = true;
         // Reject our predictions, as the call likely never got committed, so it will never be written
-        pathValueCommitter.ingestValidStates([{
-            path: pathResultWrite,
-            isValid: false,
-            time: predictResultWrite.time,
-            isTransparent: false,
-            reason: "prediction timeout rejected",
-        }]);
+        validStateComputer.ingestValuesAndValidStates({
+            pathValues: [{
+                path: pathResultWrite,
+                value: undefined,
+                locks: [],
+                lockCount: 0,
+                valid: false,
+                time: predictResultWrite.time,
+                isTransparent: false,
+            }],
+            parentSyncs: [],
+            initialTriggers: { values: new Set(), parentPaths: new Set() },
+        });
     }
 
     setTimeout(cleanupPrediction, Querysub.PREDICTION_MAX_LIFESPAN);
@@ -357,7 +367,7 @@ const getDevFunctionCache = cacheAsyncLimitedJSON(100_000, getDevFunctionSpecFro
 const addToModuleLoaderCache = cacheAsyncLimitedJSON(100_000, addModuleToLoader);
 
 function getFunctionSpec(call: CallSpec): FunctionSpec | undefined {
-    if (isLocal()) {
+    if (!isPublic()) {
         let obj = getDevFunctionCache({
             DomainName: call.DomainName,
             ModuleId: call.ModuleId,

package/src/4-querysub/schemaHelpers.ts

@@ -1,5 +1,4 @@
 import { lazy } from "socket-function/src/caching";
-import { LOCAL_DOMAIN } from "../0-path-value-core/NodePathAuthorities";
 import { registerSchemaPrefix } from "../2-proxy/PathValueProxyWatcher";
 import { rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { Schema2, Schema2T } from "../2-proxy/schema2";
@@ -7,6 +6,11 @@ import { isDynamicallyLoading } from "../config";
 import { getPathStr2 } from "../path";
 import { isHotReloading } from "socket-function/hot/HotReloadController";
 import debugbreak from "debugbreak";
+import { LOCAL_DOMAIN } from "../0-path-value-core/PathRouter";
+if (!registerSchemaPrefix) {
+    debugbreak(2);
+    debugger;
+}
 
 let localSchemaNames = new Set<string>();
 export function createLocalSchema<T>(name: string): () => T;

package/src/5-diagnostics/GenericFormat.tsx

@@ -31,7 +31,7 @@ type StringFormatters = (
     | "<Selector>"
 );
 
-function d(value: unknown, formattedValue: preact.ComponentChild) {
+function def(value: unknown, formattedValue: preact.ComponentChild) {
     if (value === undefined || value === null) {
         return "";
     }
@@ -40,14 +40,14 @@ function d(value: unknown, formattedValue: preact.ComponentChild) {
 const startGuessDateRange = +new Date(2010, 0, 1).getTime();
 const endGuessDateRange = +new Date(2040, 0, 1).getTime();
 let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.ComponentChild } = {
-    string: (value) => d(value, String(value || "")),
-    number: (value) => d(value, formatNumber(Number(value))),
-    percent: (value) => d(value, formatPercent(Number(value))),
-    timeSpan: (value) => d(value, formatTime(Number(value))),
-    date: (value) => d(value, <span title={formatDateTimeDetailed(Number(value))}>{formatDateTime(Number(value))}</span>),
-    error: (value) => d(value, <span class={errorMessage}>{String(value)}</span>),
-    toSpaceCase: (value) => d(value, toSpaceCase(String(value))),
-    "<Selector>": (value) => d(value, <Selector {...JSON.parse(String(value).slice("<Selector>".length))} />),
+    string: (value) => def(value, String(value || "")),
+    number: (value) => def(value, formatNumber(Number(value))),
+    percent: (value) => def(value, formatPercent(Number(value))),
+    timeSpan: (value) => def(value, formatTime(Number(value))),
+    date: (value) => def(value, <span title={formatDateTimeDetailed(Number(value))}>{formatDateTime(Number(value))}</span>),
+    error: (value) => def(value, <span class={errorMessage}>{String(value)}</span>),
+    toSpaceCase: (value) => def(value, toSpaceCase(String(value))),
+    "<Selector>": (value) => def(value, <Selector {...JSON.parse(String(value).slice("<Selector>".length))} />),
     link: (value) => {
         if (value === undefined || value === null) {
             return "";
@@ -118,6 +118,11 @@ let formatters: { [formatter in StringFormatters]: (value: unknown) => preact.Co
         if (Array.isArray(value) && !value.some(x => canHaveChildren(x))) {
             return formatValue(value, "varray:guess");
         }
+        if (canHaveChildren(value)) {
+            try {
+                return JSON.stringify(value);
+            } catch { }
+        }
         return formatters.string(value);
     },
 };

package/src/archiveapps/archiveGCEntry.tsx

@@ -8,6 +8,10 @@ import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 
 import yargs from "yargs";
 import { runAliveCheckerIteration } from "../2-proxy/garbageCollection";
+import { waitForImportBlockers } from "../3-path-functions/pathFunctionLoader";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
+import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("watch", { type: "boolean", desc: "Watch, and GC every N time (every day as of writing this)" })
@@ -16,12 +20,13 @@ let yargObj = isNodeTrue() && yargs(process.argv)
 
 // yarn gc --authority testshard.json
 async function main() {
-    await Querysub.hostService("gc");
-
+    disablePathAuditer();
     let folderRoot = path.resolve(".");
     folderRoot = folderRoot.replaceAll(/\\/g, "/");
     const deployPath = path.resolve("./deploy.ts");
     require(deployPath);
+    await waitForImportBlockers();
+    await Querysub.hostService("gc");
 
     if (yargObj.watch) {
         await runInfinitePollCallAtStart(timeInDay, runAliveCheckerIteration);
@@ -29,6 +34,8 @@ async function main() {
         try {
             // Force, as they are running this manually, and so they probably want to see something happen...
             await runAliveCheckerIteration({ force: true });
+        } catch (error) {
+            console.error(error);
         } finally {
             process.exit();
         }

package/src/archiveapps/archiveJoinEntry.ts

@@ -1,8 +1,6 @@
 import "../inject";
 
 import { logErrors } from "../errors";
-import { getOurAuthorities } from "../config2";
-import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
 import { FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -13,111 +11,116 @@ import { formatNumber } from "socket-function/src/formatting/format";
 import { sort } from "socket-function/src/misc";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { magenta } from "socket-function/src/formatting/logColors";
+import { PathRouter } from "../0-path-value-core/PathRouter";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
 
 async function runGenesisJoinIteration() {
-    let authorities = getOurAuthorities();
-    for (let authority of authorities) {
-        let locker = await pathValueArchives.getArchiveLocker();
-        let matchedDirs = await pathValueArchives.getAuthorityDirs(authority);
-        let authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
-        let maxAge = Date.now() - VALUE_GC_THRESHOLD;
-
-        let readCache = new Map<string, Buffer>();
-        let reread = true;
-        while (reread) {
-            reread = false;
-            let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
-                let originalCount = valueFiles.length;
-                valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
-                let underPathCount = valueFiles.length;
-                // Merge the newest first, so if we hit a big file, we can just ignore it,
-                //  and next time merge the smaller files after it.
-                sort(valueFiles, x => -x.createTime);
-                valueFiles = valueFiles.filter(x => {
-                    let obj = pathValueArchives.decodeDataPath(x.file);
-                    if (!obj.minTime) return false;
-                    if (obj.sourceType !== "genesis") return false;
-                    return x.createTime >= maxAge;
-                });
-                let withinTimeRangeCount = valueFiles.length;
-
-                if (valueFiles.length < 3) return [];
-
-                await measureBlock(async function locker_readFiles() {
-                    let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
-                    let buffers = await readFiles(remainingFiles);
-                    for (let i = 0; i < remainingFiles.length; i++) {
-                        let buffer = buffers[i];
-                        if (!buffer) {
-                            console.log(`File missing, re-reading`, remainingFiles[i].file);
-                            reread = true;
-                            break;
-                        }
-                        readCache.set(remainingFiles[i].file, buffer);
-                    }
-                });
-                if (reread) {
-                    console.log(`Files changed, re-reading`);
-                    return [];
-                }
+    let authoritySpec = await getOurAuthoritySpec(true);
+    let locker = await pathValueArchives.getArchiveLocker();
+    let maxAge = Date.now() - VALUE_GC_THRESHOLD;
 
-                let usedFiles: FileInfo[] = [];
-                let totalSize = 0;
-                let totalCount = 0;
-                let allValues: PathValue[][] = [];
-                for (let valueFile of valueFiles) {
-                    let buffer = readCache.get(valueFile.file)!;
-                    if (totalSize + buffer.length > FILE_SIZE_LIMIT) {
-                        break;
-                    }
-                    let newValues = await PathValueArchives.loadValuesFromBuffer({
-                        path: valueFile.file,
-                        data: buffer,
-                    });
+    let readCache = new Map<string, Buffer>();
+    let reread = true;
+    while (reread) {
+        reread = false;
+        let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
+            let originalCount = valueFiles.length;
+            valueFiles = valueFiles.filter(x => PathRouter.overlapsPathIdentifier(authoritySpec, x.file));
+            let underPathCount = valueFiles.length;
+            // Merge the newest first, so if we hit a big file, we can just ignore it,
+            //  and next time merge the smaller files after it.
+            sort(valueFiles, x => -x.createTime);
+            valueFiles = valueFiles.filter(x => {
+                let obj = pathValueArchives.decodeDataPath(x.file);
+                if (!obj.minTime) return false;
+                if (obj.sourceType !== "genesis") return false;
+                return x.createTime >= maxAge;
+            });
+            let withinTimeRangeCount = valueFiles.length;
+
+            if (valueFiles.length < 3) return [];
 
-                    let readValues = newValues.values;
-                    if (readValues.length + totalCount > FILE_VALUE_COUNT_LIMIT) {
+            await measureBlock(async function locker_readFiles() {
+                let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
+                let buffers = await readFiles(remainingFiles);
+                for (let i = 0; i < remainingFiles.length; i++) {
+                    let buffer = buffers[i];
+                    if (!buffer) {
+                        console.log(`File missing, re-reading`, remainingFiles[i].file);
+                        reread = true;
                         break;
                     }
-                    allValues.push(readValues);
-                    usedFiles.push(valueFile);
-                    totalSize += buffer.length;
-                    totalCount += readValues.length;
+                    readCache.set(remainingFiles[i].file, buffer);
+                }
+            });
+            if (reread) {
+                console.log(`Files changed, re-reading`);
+                return [];
+            }
+
+            let usedFiles: FileInfo[] = [];
+            let totalSize = 0;
+            let totalCount = 0;
+            let allValues: PathValue[][] = [];
+            for (let valueFile of valueFiles) {
+                let buffer = readCache.get(valueFile.file)!;
+                if (totalSize + buffer.length > FILE_SIZE_LIMIT) {
+                    break;
                 }
-                if (usedFiles.length <= 1) return [];
+                let newValues = await PathValueArchives.loadValuesFromBuffer({
+                    path: valueFile.file,
+                    data: buffer,
+                });
+
+                let readValues = newValues.values;
+                if (readValues.length + totalCount > FILE_VALUE_COUNT_LIMIT) {
+                    break;
+                }
+                allValues.push(readValues);
+                usedFiles.push(valueFile);
+                totalSize += buffer.length;
+                totalCount += readValues.length;
+            }
+            if (usedFiles.length <= 1) return [];
+
+            let allCombinedValues = allValues.flat();
 
-                let allCombinedValues = allValues.flat();
 
-                let dataObj = await pathValueArchives.encodeValuePaths(allCombinedValues, {
+            let transaction: ArchiveTransaction = {
+                createFiles: [],
+                deleteFiles: [],
+            };
+            for (let file of usedFiles) {
+                transaction.deleteFiles.push(file);
+            }
+
+            let targets = PathRouter.getSelfPathIdentifierTargets(allCombinedValues);
+            for (let [target, values] of targets) {
+
+                let dataObj = await pathValueArchives.encodeValuePaths(values, {
                     pathOverrides: {
                         sourceType: "join",
                     },
                 });
-                if (!dataObj) return [];
-                let transaction: ArchiveTransaction = {
-                    createFiles: [],
-                    deleteFiles: [],
-                };
-                for (let file of usedFiles) {
-                    transaction.deleteFiles.push(file);
-                }
-                let newFileName = authorityDir + dataObj?.file;
+                if (!dataObj) continue;
+                let newFileName = target + "/" + dataObj?.file;
                 transaction.createFiles.push({
                     file: newFileName,
                     data: dataObj.data,
                 });
+            }
 
+            console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));
 
-                console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));
-
-                return [transaction];
-            });
-            console.log(`Join result: ${result}`);
-        }
+            return [transaction];
+        });
+        console.log(`Join result: ${result}`);
     }
 }
 
 async function main() {
+    disablePathAuditer();
     await Querysub.hostService("join");
 
     await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);

package/src/archiveapps/archiveMergeEntry.tsx

@@ -5,6 +5,7 @@ import { logErrors } from "../errors";
 import { timeInDay, timeInHour } from "socket-function/src/misc";
 import { runArchiveMover } from "../2-proxy/archiveMoveHarness";
 import { PathValue, compareTime } from "../0-path-value-core/pathValueCore";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
 
 const MERGE_DELAY = timeInHour;
 
@@ -34,6 +35,7 @@ async function mergeFiles() {
 }
 
 async function main() {
+    disablePathAuditer();
     await Querysub.hostService("merge");
     await mergeFiles();
     process.exit();

package/src/bits.ts

@@ -53,6 +53,25 @@ export function setLowHighUint32(low: number, high: number): number {
     return conversionBuffer[0];
 }
 
+/** Adds protection against NaN values, changing the result if it would be NaN. This is because NaN values will compare to be not equal even if they are equal in certain cases, and in other cases, they'll always be equal, even if their bits are not equal.
+    - This means this is not a reversible operation. However, in a lot of cases, that doesn't matter. 
+*/
+export function setLowHighUint32Safe(low: number, high: number): number {
+    conversionUint32Buffer[0] = low;
+    // Prevent NaN by not setting all the exponent bits to 1
+    conversionUint32Buffer[1] = high & 0xBFFFFFFF;
+    return conversionBuffer[0];
+}
+
+// NOTE: Not reversible, see setLowHighUint32Safe
+export function xor64BitsSafe(a: number, b: number): number {
+    let high = getHighUint32(a);
+    let low = getLowUint32(a);
+    let high2 = getHighUint32(b);
+    let low2 = getLowUint32(b);
+    return setLowHighUint32Safe(low ^ low2, high ^ high2);
+}
+
 // Gets bits that can be stored in a number. Specifically, the first 62 bits,
 //  as 64 bits will not compare correctly when treated as a double.
 export function getShortNumber(buffer: Buffer): number {

package/src/config.ts

@@ -13,7 +13,9 @@ let yargObj = parseArgsFactory()
     .option("domain", { type: "string", desc: `Sets the domain` })
     .option("emaildomain", { type: "string", desc: `Sets the domain to use for email` })
     .option("client", { type: "boolean", desc: `Drops permissions, acting as an unauthenticated node` })
-    .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
+    .option("arange", { type: "string", desc: `Authority range. Ex, 0.5-1.0` })
+    .option("aexcludedefault", { type: "boolean", desc: `Exclude default authority values, so we only match our prefixes.` })
+    .option("aprefix", { type: "array", desc: `Override the prefixes (pass multiple arguments to add multiple). Can be used with aexcludedefault to restrict an authority to only a few specific prefixes (ex, ` })
     .option("nobreak", { type: "boolean", desc: "Do not break on errors. Safer to set this than to just not set debugbreak, as some places might break without checking debugbreak, but nobreak works at a level where it is always used." })
     .option("public", { type: "boolean", desc: "Expose on public ports." })
     .option("local", { type: "boolean", desc: `If true, uses the local directory instead of the remote git repo. Also hotreloads from disk. Determines the repo to replace through the package.json "repository" property.` })
@@ -27,6 +29,7 @@ let yargObj = parseArgsFactory()
         // NOTE: I wanna see how long I can keep this on for. Eventually it's gonna become a problem and we're gonna have to turn it off. But for testing it's certainly useful as we don't know exactly what is gonna cause a problem. But it probably will be synchronization related, and every server does synchronization.
         desc: "Track all audit logs to disk. This might end up writing A LOT of data."
     })
+    .option("logbackblaze", { type: "boolean", desc: "Log all backblaze activity to disk." })
     .argv
     ;
 type QuerysubConfig = {
@@ -48,6 +51,10 @@ let querysubConfig = lazy((): QuerysubConfig => {
     }
 });
 
+export function isLogBackblaze() {
+    return !!yargObj.logbackblaze;
+}
+
 export function isNoNetwork() {
     return yargObj.nonetwork;
 }
@@ -102,8 +109,18 @@ export function enableDebugging() {
 (globalThis as any).enableDebugging = enableDebugging;
 
 
-export function authorityRaw() {
-    return yargObj.authority;
+export function getAuthorityRange() {
+    return yargObj.arange;
+}
+
+export function getAuthorityExcludeDefault() {
+    return !!yargObj.aexcludedefault;
+}
+
+export function getAuthorityPrefix() {
+    if (!yargObj.aprefix) return undefined;
+    if (Array.isArray(yargObj.aprefix)) return yargObj.aprefix;
+    return [yargObj.aprefix];
 }
 
 export function isPublic() {
@@ -113,6 +130,7 @@ export function isPublic() {
     return !!yargObj.public;
 }
 
+/** @deprecated Use !isPublic() instead */
 export function isLocal() {
     return !!yargObj.local;
 }