querysub 0.682.0 → 0.684.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.682.0",
+  "version": "0.684.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/2-proxy/archiveMoveHarness.ts

@@ -297,7 +297,7 @@ export async function runArchiveMover(config: {
     console.log(" ");
 }
 
-const splitValuesBelowLimit = measureWrap(function splitValuesBelowLimit(config: {
+export const splitValuesBelowLimit = measureWrap(function splitValuesBelowLimit(config: {
     values: PathValue[];
     maxFileValueCount: number;
     maxFileByteCount: number;

package/src/archiveapps/archiveJoinEntry.ts

@@ -2,7 +2,7 @@ import "../inject";
 
 import { logErrors } from "../errors";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
-import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_STABLE_THRESHOLD, PathValue } from "../0-path-value-core/pathValueCore";
+import { ARCHIVE_FLUSH_LIMIT, ARCHIVE_STABLE_THRESHOLD, FILE_SIZE_LIMIT, FILE_VALUE_COUNT_LIMIT, PathValue } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -14,6 +14,7 @@ import { magenta } from "socket-function/src/formatting/logColors";
 import { PathRouter } from "../0-path-value-core/PathRouter";
 import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
 import { getOurAuthoritySpec } from "../0-path-value-core/PathRouterServerAuthoritySpec";
+import { splitValuesBelowLimit } from "../2-proxy/archiveMoveHarness";
 import yargs from "yargs";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
@@ -101,18 +102,23 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
 
             let targets = await PathRouter.getPathIdentifierTargets(allCombinedValues, authoritySpec);
             for (let [target, values] of targets) {
-
-                let dataObj = await pathValueArchives.encodeValuePaths(values, {
-                    pathOverrides: {
-                        sourceType: "join",
-                    },
-                });
-                if (!dataObj) continue;
-                let newFileName = target + "/" + dataObj?.file;
-                transaction.createFiles.push({
-                    file: newFileName,
-                    data: dataObj.data,
+                let split = splitValuesBelowLimit({
+                    values,
+                    maxFileValueCount: FILE_VALUE_COUNT_LIMIT,
+                    maxFileByteCount: FILE_SIZE_LIMIT,
                 });
+                for (let splitValues of split) {
+                    let dataObj = await pathValueArchives.encodeValuePaths(splitValues.values, {
+                        pathOverrides: {
+                            sourceType: "join",
+                        },
+                    });
+                    if (!dataObj) continue;
+                    transaction.createFiles.push({
+                        file: target + "/" + dataObj.file,
+                        data: dataObj.data,
+                    });
+                }
             }
 
             console.log(magenta(`Joining ${formatNumber(usedFiles.length)} => ${transaction.createFiles.length} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));

package/src/deployManager/components/storage/storageLogMCPSearch.ts

@@ -14,6 +14,7 @@ module.hotreload = false;
 
 const PARALLEL_FILE_SEARCHES = 4;
 const DEFAULT_LIMIT = 100;
+const MAX_LIMIT = 10_000;
 // Compressed input bytes of downloaded files to keep resident.
 const DOWNLOAD_CACHE_MAX_BYTES = 1024 ** 3;
 // Decoded output bytes (estimated via serialized length) of decoded files to keep resident. Sized independently of the download cache because the decoded form is much larger than the compressed download.
@@ -125,6 +126,9 @@ export async function searchStorageLogsForMCP(config: {
         throw new Error(`startTime (${startTime}) must be < endTime (${endTime})`);
     }
     let limit = config.limit ?? DEFAULT_LIMIT;
+    if (limit > MAX_LIMIT) {
+        throw new Error(`limit ${limit} > ${MAX_LIMIT}. This endpoint returns projected rows over a socket, it is not a bulk export. If you need to aggregate large volumes, list the log files and download + parse them directly.`);
+    }
     let direction = config.direction ?? "fromEnd";
     let columns = config.columns && (config.columns.includes("time") ? config.columns : ["time", ...config.columns]);
     let clauses = parseSearchClauses(query);