querysub 0.152.0 → 0.154.0

package/src/4-querysub/QuerysubController.ts

@@ -5,7 +5,7 @@ import { CHILD_CHECK_PREFIX, FunctionMetadata } from "../3-path-functions/syncSc
 import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { getProxyPath } from "../2-proxy/pathValueProxy";
 import { atomic, atomicObjectWrite, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { CallSpec, DEPTH_TO_DATA, FunctionSpec, commitCall, debugCallSpec, functionSchema } from "../3-path-functions/PathFunctionRunner";
+import { CallSpec, DEPTH_TO_DATA, FunctionSpec, commitCall, debugCallSpec, functionSchema, getDevFunctionSpecFromCall } from "../3-path-functions/PathFunctionRunner";
 import { PathValueController } from "../0-path-value-core/PathValueController";
 import { epochTime, MAX_ACCEPTED_CHANGE_AGE, PathValue, pathWatcher, Time, WatchConfig, compareTime, debugTime, getNextTime, MAX_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
 import { IdentityController_getMachineId, IdentityController_getPubKeyShort, IdentityController_getSecureIP } from "../-c-identity/IdentityController";
@@ -43,6 +43,7 @@ setFlag(require, "preact", "allowclient", true);
 import yargs from "yargs";
 import { mergeFilterables, parseFilterable, serializeFilterable } from "../misc/filterable";
 import { isManagementUser } from "../-0-hooks/hooks";
+import { isLocal } from "../config";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
@@ -580,10 +581,15 @@ export class QuerysubControllerBase {
     }
 
     public async getModulePath(config: {
-        functionSpec: FunctionSpec;
+        functionSpec: {
+            DomainName: string;
+            ModuleId: string;
+            FunctionId: string;
+        };
     }): Promise<string> {
         let spec = config.functionSpec;
         Querysub.assertDomainAllowed(getPathStr1(spec.DomainName));
+
         let moduleConfig = await proxyWatcher.commitFunction({
             watchFunction: function getModuleConfig() {
                 return functionSchema()[spec.DomainName].PathFunctionRunner[spec.ModuleId].Sources[spec.FunctionId];
@@ -596,6 +602,19 @@ export class QuerysubControllerBase {
         let module = await getModuleFromConfig(moduleConfig);
         return module.filename;
     }
+    public async getDevFunctionSpecFromCall(call: {
+        DomainName: string;
+        ModuleId: string;
+        FunctionId: string;
+    }) {
+        let functionSpec = getDevFunctionSpecFromCall(call);
+        if (!functionSpec) return undefined;
+        let modulePath = (await getModuleFromConfig(functionSpec)).filename;
+        return {
+            functionSpec,
+            modulePath,
+        };
+    }
 
 
     // NOTE: Maybe remove this? It's very useful for debugging though...
@@ -619,6 +638,7 @@ export const QuerysubController = SocketFunction.register(
         debugGetSingleReadNode: {},
         debugGetPathAuthorities: {},
         getModulePath: {},
+        getDevFunctionSpecFromCall: {},
     }),
     () => ({
 

package/src/4-querysub/permissions.ts

@@ -2,7 +2,7 @@ import { cache, cacheArgsEqual, cacheLimited } from "socket-function/src/caching
 import { measureFnc, measureWrap, nameFunction } from "socket-function/src/profiling/measure";
 import { getPathSuffix, getPathDepth, trimPathStrToDepth, getPathFromStr, rootPathStr, getPathIndex, getPathStr1, joinPathStres, appendToPathStr, getPathStr } from "../path";
 import { atomic, atomicObjectRead, isSynced, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
-import { SchemaObject, getSchemaObject, hasWildcardMatch, getWildcardMatches, PERMISSIONS_FUNCTION_ID, PermissionsCheckResult } from "../3-path-functions/syncSchema";
+import { SchemaObject, getSchemaObject, hasWildcardMatch, getWildcardMatches, PERMISSIONS_FUNCTION_ID, PermissionsCheckResult, getDevelopmentModule } from "../3-path-functions/syncSchema";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import { getSchemaPartsFromPath, functionSchema, DEPTH_TO_DATA, overrideCurrentCall, CallSpec, FunctionSpec, DOMAIN_INDEX, MODULE_INDEX } from "../3-path-functions/PathFunctionRunner";
 import debugbreak from "debugbreak";
@@ -13,6 +13,7 @@ import { isClient } from "../config2";
 import { sort } from "socket-function/src/misc";
 import { Querysub } from "./QuerysubController";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
+import { getDomain, isLocal } from "../config";
 
 function watchModule(config: FunctionSpec): NodeJS.Module | undefined {
     let module = getModuleFromConfig(config);
@@ -50,7 +51,37 @@ export class PermissionsCheck {
         );
     });
 
-    private getModulePermissions = cache((domainName: string) => {
+    private getModulePermissions(domainName: string) {
+        if (isLocal() && domainName === getDomain()) {
+            return function (moduleId: string): {
+                schema: SchemaObject;
+                fnc: FunctionSpec;
+            } | undefined {
+                let fnc: FunctionSpec = {
+                    DomainName: domainName,
+                    ModuleId: moduleId,
+                    FunctionId: PERMISSIONS_FUNCTION_ID,
+                    exportPathStr: callPermissionsPath,
+                    // NOTE: These SHOULDN'T be required, as we don't commit this function to the FunctionRunner,
+                    //  we just use this to run the function on the local machine.
+                    FilePath: "LOCAL_PERMISSIONS_HACK",
+                    gitURL: "LOCAL_PERMISSIONS_HACK",
+                    gitRef: "LOCAL_PERMISSIONS_HACK",
+                };
+                let modulePermissions = getDevelopmentModule(moduleId);
+                if (!modulePermissions) throw new Error(`No development module found for ${moduleId}. Was it imported? Try restarting the edge server.`);
+                let schema = getSchemaObject(modulePermissions);
+                if (!schema) throw new Error(`Module does not export a schema: ${moduleId}`);
+                return {
+                    schema,
+                    fnc,
+                };
+            };
+        }
+        return this.getModulePermissionsBase(domainName);
+    }
+
+    private getModulePermissionsBase = cache((domainName: string) => {
         return cache((moduleId: string) => {
             let functionConfigProxy = PermissionsCheck.skipPermissionsChecks(() =>
                 functionSchema()[domainName].PathFunctionRunner[moduleId].Sources[PERMISSIONS_FUNCTION_ID],

package/src/4-querysub/querysubPrediction.ts

@@ -20,29 +20,46 @@ import cbor from "cbor-x";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
 import { isNode, nextId, sort } from "socket-function/src/misc";
 import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
+import { isLocal } from "../config";
 setFlag(require, "cbor-x", "allowclient", true);
 const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
 
-// TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
-//  but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
-//  do too much for us if we already have the fully resolved path...
-//  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
-const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
+
+// NOTE: Most functions won't use this, as they should use regular api calls. However,
+//  as we are using paths for RequireJS, we explicitly need our local (when serverside).
+async function getPredictController() {
     let controller: QuerysubControllerBase;
     if (isNode()) {
         // NOTE: If we are on node, then the require WON'T run over the network, so we need to use
-        //  our local module path, not the remove one.
+        //  our local module path, not the remote one.
         controller = new QuerysubControllerBase();
     } else {
         let nodeId = await querysubNodeId();
         if (!nodeId) throw new Error("No querysub node found");
         controller = QuerysubController.nodes[nodeId] as any;
     }
+    return controller;
+}
+// TODO: I think our use of filePath, moduleId, etc are wrong here? We give pathFunctionLoader the filePath,
+//  but we really should just give it the moduleId, or... even just avoid calling it altogether, as it doesn't
+//  do too much for us if we already have the fully resolved path...
+//  - Although using it DOES allow permissions checks to work nicely, so, eh... maybe it is fine to use pathFunctionLoader?
+const addModuleToLoader = cacheJSONArgsEqual(async (spec: FunctionSpec): Promise<void> => {
+    let controller = await getPredictController();
 
     let path = await controller.getModulePath({ functionSpec: spec });
     setGitURLMapping({ spec, resolvedPath: path });
 });
 
+const getDevFunctionSpecFromCall = cacheJSONArgsEqual(async (call: {
+    DomainName: string;
+    ModuleId: string;
+    FunctionId: string;
+}) => {
+    let controller = await getPredictController();
+    return controller.getDevFunctionSpecFromCall(call);
+});
+
 
 export function getCallResultPath(call: CallSpec) {
     return getProxyPath(() => functionSchema()[call.DomainName].PathFunctionRunner[call.ModuleId].Results[call.CallId]);
@@ -490,20 +507,36 @@ export async function getCallWrites(config: {
     overrides?: PathValue[];
 }) {
     let { call, debugName } = config;
-    const { functionSpec } = await proxyWatcher.commitFunction({
-        watchFunction: function getModuleConfig() {
-            let domainObject = functionSchema()[call.DomainName];
-            let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
-            let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
-            return { functionSpec };
+
+    let functionSpec: FunctionSpec | undefined;
+    if (isLocal()) {
+        let obj = await getDevFunctionSpecFromCall({
+            DomainName: call.DomainName,
+            ModuleId: call.ModuleId,
+            FunctionId: call.FunctionId,
+        });
+        if (!obj) throw new Error(`Function not referenced in deploy.ts ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
+        functionSpec = obj.functionSpec;
+        setGitURLMapping({ spec: functionSpec, resolvedPath: obj.modulePath });
+    } else {
+        const obj = await proxyWatcher.commitFunction({
+            watchFunction: function getModuleConfig() {
+                let domainObject = functionSchema()[call.DomainName];
+                let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
+                let functionSpec = atomicObjectRead(moduleObject.Sources[call.FunctionId]);
+                return { functionSpec };
+            }
+        });
+        functionSpec = obj.functionSpec;
+
+        if (!functionSpec) {
+            throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
         }
-    });
-    if (!functionSpec) {
-        throw new Error(`Function not found in database ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
+        // Add the module to the loader via asking the server the exact url for this call. The loader will
+        //  then load the code from that url when it ends up running it.
+        await addModuleToLoader(functionSpec);
     }
 
-    await addModuleToLoader(functionSpec);
-
     let module = await getModuleFromConfig(functionSpec);
     let exportPath = getPathFromStr(functionSpec.exportPathStr);
     let exportObj = module.exports;
@@ -515,6 +548,7 @@ export async function getCallWrites(config: {
     }
     let baseFunction = exportObj as Function;
 
+    let specTyped = functionSpec;
     return await proxyWatcher.dryRunFull({
         debugName,
         runAtTime: call.runAtTime,
@@ -524,7 +558,7 @@ export async function getCallWrites(config: {
         overrides: config.overrides,
         nestedCalls: "inline",
         watchFunction() {
-            return overrideCurrentCall({ spec: call, fnc: functionSpec }, () => {
+            return overrideCurrentCall({ spec: call, fnc: specTyped }, () => {
                 let args = parseArgs(call);
                 return baseFunction(...args);
             });

package/src/archiveapps/archiveGCEntry.tsx

@@ -0,0 +1,38 @@
+import "querysub/inject";
+
+import { Querysub } from "../4-querysub/QuerysubController";
+import { logErrors } from "../errors";
+import { isNodeTrue, timeInDay, timeInHour } from "socket-function/src/misc";
+import path from "path";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+
+import yargs from "yargs";
+import { runAliveCheckerIteration } from "../2-proxy/garbageCollection";
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("watch", { type: "boolean", desc: "Watch, and GC every N time (every day as of writing this)" })
+    .argv || {}
+    ;
+
+// yarn gc --authority testshard.json
+async function main() {
+    await Querysub.hostService("gc");
+
+    let folderRoot = path.resolve(".");
+    folderRoot = folderRoot.replaceAll(/\\/g, "/");
+    const deployPath = path.resolve("./deploy.ts");
+    require(deployPath);
+
+    if (yargObj.watch) {
+        await runInfinitePollCallAtStart(timeInDay, runAliveCheckerIteration);
+    } else {
+        try {
+            // Force, as they are running this manually, and so they probably want to see something happen...
+            await runAliveCheckerIteration({ force: true });
+        } finally {
+            process.exit();
+        }
+    }
+}
+
+main().catch(logErrors);

package/src/archiveapps/archiveJoinEntry.ts

@@ -0,0 +1,121 @@
+import "querysub/inject";
+
+import { logErrors } from "../errors";
+import { getOurAuthorities } from "../config2";
+import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
+import { FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { ArchiveTransaction, FileInfo } from "../0-path-value-core/archiveLocks/ArchiveLocks";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { sort } from "socket-function/src/misc";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { magenta } from "socket-function/src/formatting/logColors";
+
+async function runGenesisJoinIteration() {
+    let authorities = getOurAuthorities();
+    for (let authority of authorities) {
+        let locker = await pathValueArchives.getArchiveLocker();
+        let matchedDirs = await pathValueArchives.getAuthorityDirs(authority);
+        let authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
+        let maxAge = Date.now() - VALUE_GC_THRESHOLD;
+
+        let readCache = new Map<string, Buffer>();
+        let reread = true;
+        while (reread) {
+            reread = false;
+            let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
+                valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
+
+                // Merge the newest first, so if we hit a big file, we can just ignore it,
+                //  and next time merge the smaller files after it.
+                sort(valueFiles, x => -x.createTime);
+                valueFiles = valueFiles.filter(x => {
+                    let obj = pathValueArchives.decodeDataPath(x.file);
+                    if (!obj.minTime) return false;
+                    if (obj.sourceType !== "genesis") return false;
+                    return x.createTime >= maxAge;
+                });
+
+                if (valueFiles.length < 3) return [];
+
+                await measureBlock(async function locker_readFiles() {
+                    let remainingFiles = valueFiles.filter(x => !readCache.has(x.file));
+                    let buffers = await readFiles(remainingFiles);
+                    for (let i = 0; i < remainingFiles.length; i++) {
+                        let buffer = buffers[i];
+                        if (!buffer) {
+                            console.log(`File missing, re-reading`, remainingFiles[i].file);
+                            reread = true;
+                            break;
+                        }
+                        readCache.set(remainingFiles[i].file, buffer);
+                    }
+                });
+                if (reread) {
+                    console.log(`Files changed, re-reading`);
+                    return [];
+                }
+
+                let usedFiles: FileInfo[] = [];
+                let totalSize = 0;
+                let totalCount = 0;
+                let allValues: PathValue[][] = [];
+                for (let valueFile of valueFiles) {
+                    let buffer = readCache.get(valueFile.file)!;
+                    if (totalSize + buffer.length > FILE_SIZE_LIMIT) {
+                        break;
+                    }
+                    let newValues = await PathValueArchives.loadValuesFromBuffer({
+                        path: valueFile.file,
+                        data: buffer,
+                    });
+                    let readValues = newValues.values;
+                    if (readValues.length + totalCount > FILE_VALUE_COUNT_LIMIT) {
+                        break;
+                    }
+                    allValues.push(readValues);
+                    usedFiles.push(valueFile);
+                    totalSize += buffer.length;
+                    totalCount += readValues.length;
+                }
+                if (usedFiles.length <= 1) return [];
+
+                let allCombinedValues = allValues.flat();
+
+                console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes`));
+
+                let dataObj = await pathValueArchives.encodeValuePaths(allCombinedValues, {
+                    pathOverrides: {
+                        sourceType: "join",
+                    },
+                });
+                if (!dataObj) return [];
+                let transaction: ArchiveTransaction = {
+                    createFiles: [],
+                    deleteFiles: [],
+                };
+                for (let file of usedFiles) {
+                    transaction.deleteFiles.push(file);
+                }
+                transaction.createFiles.push({
+                    file: authorityDir + dataObj?.file,
+                    data: dataObj.data,
+                });
+
+                return [transaction];
+            });
+            console.log(`Join result: ${result}`);
+        }
+    }
+}
+
+async function main() {
+    await Querysub.hostService("join");
+
+    await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
+}
+
+main().catch(logErrors);

package/src/archiveapps/archiveMergeEntry.tsx

@@ -0,0 +1,47 @@
+import "querysub/inject";
+
+import { Querysub } from "../4-querysub/QuerysubController";
+import { logErrors } from "../errors";
+import { timeInDay, timeInHour } from "socket-function/src/misc";
+import { runArchiveMover } from "../2-proxy/archiveMoveHarness";
+import { PathValue, compareTime } from "../0-path-value-core/pathValueCore";
+
+const MERGE_DELAY = timeInHour;
+
+async function mergeFiles() {
+    await runArchiveMover({
+        outputType: "merge",
+        async runMover(config) {
+            let values = config.values;
+            let latestValues = new Map<string, PathValue>();
+            for (let value of values) {
+                let prev = latestValues.get(value.path);
+                if (prev && compareTime(value.time, prev.time) < 0) continue;
+                latestValues.set(value.path, value);
+            }
+
+            values = Array.from(latestValues.values());
+            values = values.filter(x => !x.canGCValue);
+
+            return {
+                newValues: {
+                    "": values,
+                },
+            };
+        },
+    });
+}
+
+async function main() {
+    await Querysub.hostService("merge");
+    await mergeFiles();
+    process.exit();
+
+    // while (true) {
+    //     await mergeFiles();
+    //     let delay = MERGE_DELAY * (1 + Math.random() * 0.1);
+    //     await new Promise(resolve => setTimeout(resolve, delay));
+    // }
+}
+
+main().catch(logErrors);

package/src/archiveapps/compressTest.tsx

@@ -0,0 +1,59 @@
+import fs from "fs";
+import { measureBlock, measureCode } from "socket-function/src/profiling/measure";
+import zlib from "zlib";
+const brotli = require("brotli-wasm");
+
+
+async function main() {
+    let path = "D:/repos/qs-aipaint/model4/planremapped_unet.cyberrealistic_v33.1_lora.epi_noiseoffset2.1unet_71975fc6d0215e536a96ba7eb7a44e3a503bd265ae00cfdcfebd9fe5b92f54cb.plan0.plan";
+    let data = fs.readFileSync(path);
+
+    // None of the zstd libraries work. It is about 10X faster at encoding, but... only
+    //      about 2X faster at decoding, so... whatever... it's fine for now.
+    //      It's not THAT hard to add support for other libraries later on...
+    //      (We can even have all endpoints use zip, and only have the disk read/writing use
+    //          a new format, that way only the PathValueServers have to update).
+    //  brotli was similar, except brotli-wasm, which just hung forever
+    //  I think a lot of these libraries use emscripten with bad configuration that doesn't
+    //      let it allocate more memory.
+    await measureCode(async () => {
+        /*
+        {
+            let compressed = await measureBlock(async function compressZip() {
+                return zlib.gzipSync(data);
+            });
+            let decompressed = await measureBlock(async function decompressZip() {
+                return zlib.gunzipSync(compressed);
+            });
+            console.log("Zip", compressed.length, decompressed.length);
+        }
+        */
+        // {
+
+        //     let compressed = await measureBlock(async function compressZStd() {
+        //         return zstd2.ZstdSimple.compress(data);
+        //     });
+        //     let decompressed = await measureBlock(async function decompressZStd() {
+        //         return zstd2.ZstdSimple.decompress(compressed);
+        //     });
+        //     // let decompressed2 = await measureBlock(async function decompressFZStd() {
+        //     //     return fzstd.decompress(compressed);
+        //     // });
+        //     console.log("ZStd", compressed.length, decompressed.length);
+        // }
+        {
+
+            let compressed = await measureBlock(async function compressZStd() {
+                return brotli.compress(data);
+            });
+            let decompressed = await measureBlock(async function decompressZStd() {
+                return brotli.decompress(compressed);
+            });
+            // let decompressed2 = await measureBlock(async function decompressFZStd() {
+            //     return fzstd.decompress(compressed);
+            // });
+            console.log("ZStd", compressed.length, decompressed.length);
+        }
+    });
+}
+main().catch(e => console.error(e)).finally(() => process.exit());

package/src/archiveapps/lockTest.ts

@@ -0,0 +1,127 @@
+import { pathValueArchives } from "../0-path-value-core/pathValueArchives";
+import { getPathStr, rootPathStr } from "../path";
+import { ArchiveLocker, ArchiveTransaction } from "../0-path-value-core/archiveLocks/ArchiveLocks";
+import { getDomain } from "../config";
+import { getOurAuthorities } from "../config2";
+import { getNodeId } from "socket-function/src/nodeCache";
+import { getOurNodeIdAssert, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { AuthorityPath, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { PathValue, getNextTime } from "../0-path-value-core/pathValueCore";
+import debugbreak from "debugbreak";
+import { delay } from "socket-function/src/batching";
+import { isDefined } from "../misc";
+import { sort, timeInMinute } from "socket-function/src/misc";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { formatTime } from "socket-function/src/formatting/format";
+
+// yarn typenode src\framework-beta\archivemerge\lockTest.ts --nonetwork
+async function main() {
+    await Querysub.hostService("lockTest");
+    /*
+    let allLockers: ArchiveLocker[] = [];
+    for (let authority of getOurAuthorities()) {
+        let lockers = await pathValueArchives.getRawArchives(authority);
+        allLockers.push(...lockers);
+    }
+    */
+
+    const START = 5;
+    const ITERATION_COUNT = 5;
+    const COUNT_LIMIT = ITERATION_COUNT * 5;
+
+    // D:\repos\qs-aipaint\database-storage\archives\path-values-recycle-bin\=-.,querysub.__com.,
+
+    let authority: AuthorityPath = getOurAuthorities()[0];
+    let locker = await pathValueArchives.getArchiveLocker();
+    let matchedDirs = await pathValueArchives.getAuthorityDirs(authority);
+    const authorityDir = pathValueAuthority2.getArchiveDirectory(authority);
+    while (true) {
+        let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
+            valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
+            let prevFiles = valueFiles.filter(x => pathValueArchives.decodeDataPath(x.file).sourceType === "test");
+            let prevData = (await readFiles(prevFiles)).filter(isDefined);
+
+            let pathValues: number[][] = [];
+            for (let prevBuffer of prevData) {
+                let prevValue = await pathValueSerializer.deserialize([prevBuffer], { singleBuffer: true });
+                pathValues.push(prevValue.map(x => pathValueSerializer.getPathValue(x) as number));
+            }
+            console.log(`Values [${pathValues.map(x => "[" + x.join(", ") + "]").join(", ")}]`);
+
+            let flatValues = pathValues.flat();
+            sort(flatValues, x => x);
+            let minValue = flatValues[0] || START;
+            if (flatValues.length > 0) {
+                let expectedValues = Array.from({ length: flatValues.length }, (_, i) => i + minValue);
+                if (flatValues.join(",") !== expectedValues.join(",")) {
+                    const errorMessage = `Values should be [${expectedValues.join(", ")}], was [${flatValues.join(", ")}]`;
+                    console.error(errorMessage);
+                    debugbreak(2);
+                    debugger;
+                    throw new Error(errorMessage);
+                }
+            }
+
+            const transaction: ArchiveTransaction = {
+                createFiles: [],
+                deleteFiles: [],
+            };
+            function removeValue(value: number) {
+                console.log(`Deleting file with value ${value}`);
+                let index = pathValues.findIndex(x => x[0] === value);
+                let file = prevFiles[index];
+                transaction.deleteFiles.push(file);
+            }
+            async function addValue(newValue: number) {
+                console.log(`Adding file with value ${newValue}`);
+                let newValues: PathValue[] = [];
+                newValues.push({
+                    path: getPathStr([getDomain(), "test"]),
+                    value: newValue,
+                    time: getNextTime(),
+                    locks: [],
+                    lockCount: 0,
+                    valid: true,
+                });
+                const dataObj = (await pathValueArchives.encodeValuePaths(
+                    newValues,
+                    {
+                        pathOverrides: {
+                            sourceType: "test",
+                            misc: newValue + ""
+                        },
+                    }
+                ))!;
+                transaction.createFiles.push({
+                    data: dataObj.data,
+                    file: authorityDir + dataObj.file,
+                });
+            }
+
+            let newCount = flatValues.length + ITERATION_COUNT;
+            let extraCount = newCount - COUNT_LIMIT;
+            for (let i = 0; i < extraCount; i++) {
+                removeValue(flatValues[i]);
+            }
+
+            let lastValue = flatValues[flatValues.length - 1] ?? START;
+            for (let i = 0; i < ITERATION_COUNT; i++) {
+                await addValue(lastValue + i + 1);
+            }
+
+            let timeWaitTime = Math.random() * 5 * 1000;
+            console.log(`Test wait ${formatTime(timeWaitTime)}`);
+            await delay(timeWaitTime);
+
+            return [transaction];
+        });
+        if (result === "rejected") {
+            console.warn(red("Rejected transaction"));
+        } else {
+            console.log(green("Transaction committed"));
+        }
+    }
+}
+main().catch(console.error).finally(() => process.exit());

package/src/config.ts

@@ -17,6 +17,11 @@ let yargObj = isNodeTrue() && yargs(process.argv)
     .argv || {}
     ;
 
+if (!isNode()) {
+    // TODO: Get yargs running in the browser, instead of this hack processing
+    yargObj.local = process.argv.includes("--local");
+}
+
 export function isNoNetwork() {
     return yargObj.nonetwork;
 }