querysub 0.422.0 → 0.424.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.422.0",
+  "version": "0.424.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -2,7 +2,7 @@ import { lazy } from "socket-function/src/caching";
 import { Archives, nestArchives } from "../../../-a-archives/archives";
 import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond, timeoutToUndefinedSilent } from "socket-function/src/misc";
 import { BufferIndex } from "./BufferIndex";
-import { delay, runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { delay, runInParallel, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION, mergeIndexedLogResults } from "./BufferIndexHelpers";
 import { getDomain, isPublic } from "../../../config";
 import { getArchivesHome, getArchivesLocal } from "../../../-a-archives/archivesDisk";
@@ -127,7 +127,7 @@ export class IndexedLogs<T> {
     }
 
     private runLogMoverLoop = lazy(() => {
-        runInfinitePoll(timeInMinute * 5, async () => {
+        void runInfinitePollCallAtStart(timeInMinute * 5, async () => {
             await this.moveLogsToPublic();
         });
     });

package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts

@@ -133,8 +133,37 @@ export async function moveLogsToPublic(config: {
 
     let byStartTime = keyByArray(localPaths, x => x.startTime);
 
-    let groups = Array.from(byStartTime.values());
-    sort(groups, x => x[0].startTime);
+    let initialGroups = Array.from(byStartTime.values());
+    sort(initialGroups, x => x[0].startTime);
+
+    // Get compressed file sizes for all paths in parallel to avoid loading too much data at once
+    let sizeThreshold = maxSingleFileData / 10;
+    let allPathsFlat = initialGroups.flat();
+    let pathSizes = await Promise.all(allPathsFlat.map(async path => {
+        let info = await localLogs.getInfo(path.fullPath);
+        return { path, size: info?.size || 0 };
+    }));
+    let sizeMap = new Map(pathSizes.map(x => [x.path, x.size]));
+
+    // Split individual groups if their compressed file sizes exceed threshold
+    // but keep groups separate (don't mix startTimes)
+    let groups: typeof initialGroups = [];
+    for (let group of initialGroups) {
+        let currentSize = 0;
+        
+        for (let path of group) {
+            let size = sizeMap.get(path) || 0;
+            
+            if (groups.length === 0 || currentSize + size > sizeThreshold) {
+                groups.push([]);
+                currentSize = 0;
+            }
+            
+            groups[groups.length - 1].push(path);
+            currentSize += size;
+        }
+    }
+
     for (let i = 0; i < groups.length; i++) {
         let group = groups[i];
         let time = Date.now();