qvdjs 2.2.0 → 2.2.1

package/dist/index.cjs

@@ -2265,9 +2265,10 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
   }
   return low;
 }
-function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false) {
+function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false, retainedBytes = 0) {
   const answer = checkMemory({
     wholeSymbols,
+    retainedBytes,
     symbolTableSize,
     maxRows,
     totalRows,
@@ -2300,7 +2301,8 @@ function checkMemory({
   bytesHeld = null,
   readBytes = null,
   measured = null,
-  wholeSymbols = false
+  wholeSymbols = false,
+  retainedBytes = 0
 }) {
   if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
     throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
@@ -2324,7 +2326,12 @@ function checkMemory({
     rowsLive,
     wholeSymbols
   );
-  const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
+  const {
+    held,
+    afterRelease: heldAfterRelease = null,
+    forRows: heldForRows,
+    forChunk: heldForChunk
+  } = bytesHeld ?? noBytesHeld;
   const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
   const bounded = budget.candidates.map((candidate) => {
     const heapOnly = candidate.source === "V8 heap limit";
@@ -2360,6 +2367,25 @@ function checkMemory({
   const availableMemory = binding ? binding.bytes : budget.bytes;
   const estimatedMemory = binding ? binding.needs : heapMemory;
   const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
+  const retainedIsTheReason = (() => {
+    if (binding === null || retainedBytes <= 0) {
+      return false;
+    }
+    const heapAfter = estimateMemoryUsage(
+      Math.max(0, symbolTableSize - retainedBytes),
+      maxRows,
+      totalRows,
+      columnCount,
+      materialisesRows,
+      rowsLive,
+      wholeSymbols
+    );
+    const externalAfter = estimateExternalMemory(liveRows, columnCount) + (heldAfterRelease ?? held);
+    return budget.candidates.every((candidate) => {
+      const heapOnly = candidate.source === "V8 heap limit";
+      return (heapOnly ? heapAfter : heapAfter + externalAfter) <= candidate.bytes * safetyFactor;
+    });
+  })();
   if (binding) {
     const includeExternal = !binding.heapOnly;
     const rowsHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForRows(rows) : 0), "rowsHeldBudget");
@@ -2406,13 +2432,17 @@ function checkMemory({
     const knob = chunked ? "chunkSize" : "limit";
     const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
     const nothingFits = recommendedValue === 0;
+    const held2 = retainedIsTheReason;
+    const fixedCost = held2 ? "the columns this file is holding exceed" : "the symbol table alone exceeds";
+    const release = held2 ? `Close the file and open it again to release them, or raise ` : `Raise `;
+    const releaseFirst = held2 ? `Close the file and open it again to release the columns it is holding, which is the direct remedy. ` : "";
     let advice;
     if (nothingFits) {
-      advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
+      advice = `No row count fits this budget - ${fixedCost} it, so ${knob} cannot help. ` + (containerBound ? `${release}the container's memory limit.` : `${release}the heap with --max-old-space-size, or raise memorySafetyFactor.`);
     } else if (containerBound) {
-      advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
+      advice = releaseFirst + `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
     } else {
-      advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
+      advice = releaseFirst + `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
     }
     const suggestions = [];
     if (!nothingFits) {
@@ -2436,10 +2466,16 @@ function checkMemory({
       exact: symbolTableSize === 0,
       suggestions,
       refusal: {
-        message: `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
+        message: `Insufficient memory to load file safely. ${retainedIsTheReason ? "Columns held" : "Symbol table"}: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
         context: {
           symbolTableSize,
           symbolTableSizeMB: sizeMB,
+          // Whether that figure is the file's symbol table or what an open file is still holding. A
+          // paging read is charged for every column any of its pages decoded and kept, so a caller
+          // branching on the refusal needs to know which of the two it is looking at - the remedies
+          // differ, and for this one releasing is a remedy where raising the limit is only a workaround.
+          holdsDecodedColumns: retainedIsTheReason,
+          retainedSymbolBytes: retainedBytes,
           estimatedMemoryMB: estimatedMB,
           availableMemoryMB: availableMB,
           heapLimitMB,
@@ -2607,19 +2643,39 @@ function validateHeaderStructure(headerObj, filePath, stage) {
   });
   return fieldList;
 }
-function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
+function validateSymbolTableSizeEarly(symbolTableLength, filePath, retainedBytes = 0) {
   const heapLimit = getHeapLimit();
   const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
   if (symbolTableLength > MAX_SYMBOL_TABLE_SIZE) {
     const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
     const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
     const heapMB = Math.round(heapLimit / 1024 / 1024);
+    if (retainedBytes > 0 && symbolTableLength - retainedBytes <= MAX_SYMBOL_TABLE_SIZE) {
+      throw new exports.QvdValidationError(
+        `Columns held too large (${sizeMB}MB exceeds ${maxMB}MB limit). This open file is holding the columns its pages have decoded, and they have grown past the ceiling rather than the file's own symbol table being large. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) closing the file and opening it again, which releases what the pages decoded, (2) paging over fewer columns with fields, or (3) increasing heap size with --max-old-space-size.`,
+        {
+          file: filePath,
+          symbolTableSize: symbolTableLength,
+          symbolTableSizeMB: sizeMB,
+          holdsDecodedColumns: true,
+          retainedSymbolBytes: retainedBytes,
+          maxAllowed: MAX_SYMBOL_TABLE_SIZE,
+          maxAllowedMB: maxMB,
+          heapLimitMB: heapMB,
+          reason: "memory"
+        }
+      );
+    }
     throw new exports.QvdValidationError(
       `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
       {
         file: filePath,
         symbolTableSize: symbolTableLength,
         symbolTableSizeMB: sizeMB,
+        // Present and false, not absent. A caller told it can branch on this has to find it on both
+        // refusals, or the branch reads `undefined` for the commoner of the two.
+        holdsDecodedColumns: false,
+        retainedSymbolBytes: retainedBytes,
         maxAllowed: MAX_SYMBOL_TABLE_SIZE,
         maxAllowedMB: maxMB,
         heapLimitMB: heapMB,
@@ -4025,6 +4081,10 @@ var init_QvdFileReader = __esm({
         const columnCount = selected.length;
         const symbolBytes = symbolBytesOf(this._fieldsHeldAfter(selected, headerFields), symbolTableLength);
         const readSymbolBytes = symbolBytesOf(this._fieldsReadBy(selected), symbolTableLength);
+        const retainedBytes = symbolBytesOf(
+          this._fieldsHeldAfter(selected, headerFields).slice(selected.length),
+          symbolTableLength
+        );
         const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
         const windowRows = resolved.limit;
         if (headerNumbersUsable && this._headerMatchesFile) {
@@ -4042,7 +4102,8 @@ var init_QvdFileReader = __esm({
               windowRows,
               recordSize,
               liveRows,
-              this._analysisAhead(window, resolved, totalRows, symbolTableLength)
+              this._analysisAhead(window, resolved, totalRows, symbolTableLength),
+              symbolBytes - retainedBytes
             ),
             // What it reads, which is not what it holds - the records go through one buffer and are not
             // kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
@@ -4054,7 +4115,8 @@ var init_QvdFileReader = __esm({
             // A paging read keeps whole columns, so the estimate must not discount its symbols as a window's
             // sample of them - see `estimateMemoryUsage`. Under-charging is the direction that ends in a
             // heap-limit abort rather than an error.
-            this._symbolCache !== null
+            this._symbolCache !== null,
+            retainedBytes
           );
         }
         if (window.offset === 0 && window.limit === null) {
@@ -4062,7 +4124,7 @@ var init_QvdFileReader = __esm({
           return;
         }
         const rowsToLoad = windowRows;
-        validateSymbolTableSizeEarly(symbolBytes, this._path);
+        validateSymbolTableSizeEarly(symbolBytes, this._path, retainedBytes);
         validateRecordSize(recordSize, this._path, "readData");
         validateRecordCount(totalRows, this._path, "readData");
         const fileBytesRequired = indexTableOffset + (resolved.offset + rowsToLoad) * recordSize;
@@ -4174,9 +4236,14 @@ var init_QvdFileReader = __esm({
        *   would hold.
        * @private
        */
-      _bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead) {
+      _bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead, freshSymbolBytes = null) {
         return {
           held: this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
+          // What it would hold having closed the file and opened it again: every selected column read fresh,
+          // because nothing is cached any more. Higher than `held`, not lower - a cached column this read
+          // selects costs nothing to read now and would cost its area then. Without it the counterfactual
+          // that decides whether releasing helps was answered against the warm figure and said yes too often.
+          afterRelease: freshSymbolBytes === null ? null : this._bytesHeldBy(freshSymbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
           // A window of so many rows reads so many records at a time, and the pass that reads it ahead of the
           // decode reads the same rows, so the buffer is sized from the rows either way.
           forRows: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, rows, recordSize), "forRows"),
@@ -4802,7 +4869,8 @@ var init_QvdFileReader = __esm({
         const symbolTableSize = this._symbolTableLength();
         const plan = this._symbolAreaPlan();
         const readSymbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
-        const symbolBytes = readSymbolBytes + symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
+        const retainedBytes = symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
+        const symbolBytes = readSymbolBytes + retainedBytes;
         const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
         const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
         validateSymbolTableSize(symbolBytes, this._path, totalRows);
@@ -4816,11 +4884,12 @@ var init_QvdFileReader = __esm({
             fields.length,
             this._materialisesRows,
             liveRows,
-            this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false),
+            this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false, symbolBytes - retainedBytes),
             // `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
             // run has read the window's records once already - so the read's total is two passes over them.
             readSymbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize,
-            this._symbolCache !== null
+            this._symbolCache !== null,
+            retainedBytes
           );
         }
         warnLargeSymbolTable(
@@ -5093,6 +5162,10 @@ var init_QvdFileReader = __esm({
         const ask = /* @__PURE__ */ __name((asked, rows) => {
           const bytes = symbolBytesOf(this._fieldsHeldAfter(asked, this._allFields), symbolTableLength);
           const read = symbolBytesOf(this._fieldsReadBy(asked), symbolTableLength);
+          const retained = symbolBytesOf(
+            this._fieldsHeldAfter(asked, this._allFields).slice(asked.length),
+            symbolTableLength
+          );
           return checkMemory({
             measured,
             symbolTableSize: bytes,
@@ -5104,7 +5177,8 @@ var init_QvdFileReader = __esm({
             live: liveRows,
             // A paging read keeps whole columns, so it is charged for whole columns - see `estimateMemoryUsage`.
             wholeSymbols: this._symbolCache !== null,
-            bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead),
+            retainedBytes: retained,
+            bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead, bytes - retained),
             // What it reads from the file, which is not what it holds: the symbol areas it has still to read,
             // and every record the window covers, read a slice at a time and not kept - twice over where the
             // symbol-usage pass will run, since it reads them before the decode reads them again.