querysub 0.681.0 → 0.683.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.681.0",
+  "version": "0.683.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -209,7 +209,7 @@ export function createArchiveLocker2(config: {
                 readFiles: (files: FileInfo[]) => Promise<(Buffer | undefined)[]>,
             ) => Promise<ArchiveTransaction[]>
         ): Promise<"accepted" | "rejected"> {
-            let files = await locker.getFiles();
+            let files = await locker.getFiles({ forTransaction: true });
             await saveSnapshot({ files: files.map(a => a.file) });
             let readFiles = async (files: FileInfo[]) => {
                 let pendingFiles = files.slice();
@@ -494,21 +494,23 @@ class TransactionLocker {
             let time = Date.now();
             let files = await this.storage.getKeys(config);
             this.perf.checkpoint("getKeys");
-            if (this.lastFilesRead && this.lastFilesReadTime) {
-                let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
-                let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
-                let newFiles = files.filter(a => !prevFiles.has(a.file));
-                let veryBadFiles = newFiles.filter(x => x.createTime < suspiciousThreshold);
-                if (veryBadFiles.length > 0) {
-                    console.error(`Old read was not exhaustive, because we just found files which existed well before it, but that it did not read. This likely means our propagation assumptions are wrong, and propagation is much slower than anticipated. OR, getKeys is flakey and randomly misses files? Either way, this will BREAK THE DATABASE! Any node experiencing this might start deleting valid database files, which is very bad!`, {
-                        newlyAppearingOldFiles: veryBadFiles.map(x => x.file),
-                        prevFiles: this.lastFilesRead.map(x => x.file),
-                        newFiles: files.map(x => x.file),
-                    });
+            if (!config?.fallbacks) {
+                if (this.lastFilesRead && this.lastFilesReadTime) {
+                    let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
+                    let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
+                    let newFiles = files.filter(a => !prevFiles.has(a.file));
+                    let veryBadFiles = newFiles.filter(x => x.createTime < suspiciousThreshold);
+                    if (veryBadFiles.length > 0) {
+                        console.error(`Old read was not exhaustive, because we just found files which existed well before it, but that it did not read. This likely means our propagation assumptions are wrong, and propagation is much slower than anticipated. OR, getKeys is flakey and randomly misses files? Either way, this will BREAK THE DATABASE! Any node experiencing this might start deleting valid database files, which is very bad!`, {
+                            newlyAppearingOldFiles: veryBadFiles.map(x => x.file),
+                            prevFiles: this.lastFilesRead.map(x => x.file),
+                            newFiles: files.map(x => x.file),
+                        });
+                    }
                 }
+                this.lastFilesRead = files;
+                this.lastFilesReadTime = time;
             }
-            this.lastFilesRead = files;
-            this.lastFilesReadTime = time;
 
             let transactions: (Transaction & {
                 seqNum: number;
@@ -794,7 +796,7 @@ class TransactionLocker {
     /** Only returns data files (no transaction files, or confirmations).
      *      - Might run a transaction
      */
-    public async getFiles(): Promise<FileInfo[]> {
+    public async getFiles(config?: { forTransaction?: boolean }): Promise<FileInfo[]> {
         let time = Date.now();
         let storageAlivePromise = errorToUndefinedSilent(this.storage.getSyncStatus());
         try {
@@ -805,7 +807,7 @@ class TransactionLocker {
             return obj.dataFiles;
         } catch (e) {
             let duration0 = Date.now() - time;
-            if (await storageAlivePromise) {
+            if (config?.forTransaction || await storageAlivePromise) {
                 throw e;
             }
             let duration = Date.now() - time;

package/src/2-proxy/archiveMoveHarness.ts

@@ -297,7 +297,7 @@ export async function runArchiveMover(config: {
     console.log(" ");
 }
 
-const splitValuesBelowLimit = measureWrap(function splitValuesBelowLimit(config: {
+export const splitValuesBelowLimit = measureWrap(function splitValuesBelowLimit(config: {
     values: PathValue[];
     maxFileValueCount: number;
     maxFileByteCount: number;

package/src/archiveapps/archiveJoinEntry.ts

@@ -2,7 +2,7 @@ import "../inject";
 
 import { logErrors } from "../errors";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
-import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_STABLE_THRESHOLD, PathValue } from "../0-path-value-core/pathValueCore";
+import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_STABLE_THRESHOLD, FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -14,6 +14,7 @@ import { magenta } from "socket-function/src/formatting/logColors";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
+import { splitValuesBelowLimit } from "../2-proxy/archiveMoveHarness";
 import yargs from "yargs";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -101,18 +102,23 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
 
             let targets = await PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
-
-                let dataObj = await pathValueArchives.encodeValuePaths(values, {
-                    pathOverrides: {
-                        sourceType: "join",
-                    },
-                });
-                if (!dataObj) continue;
-                let newFileName = target + "/" + dataObj?.file;
-                transaction.createFiles.push({
-                    file: newFileName,
-                    data: dataObj.data,
+                let split = splitValuesBelowLimit({
+                    values,
+                    maxFileValueCount: FILE_VALUE_COUNT_LIMIT,
+                    maxFileByteCount: FILE_SIZE_LIMIT,
                 });
+                for (let splitValues of split) {
+                    let dataObj = await pathValueArchives.encodeValuePaths(splitValues.values, {
+                        pathOverrides: {
+                            sourceType: "join",
+                        },
+                    });
+                    if (!dataObj) continue;
+                    transaction.createFiles.push({
+                        file: target + "/" + dataObj.file,
+                        data: dataObj.data,
+                    });
+                }
             }
 
             console.log(magenta(`Joining ${formatNumber(usedFiles.length)} => ${transaction.createFiles.length} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));