qvdjs 2.2.0 → 2.2.2

package/dist/index.cjs

@@ -2265,9 +2265,10 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
   }
   return low;
 }
-function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false) {
+function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false, retainedBytes = 0) {
   const answer = checkMemory({
     wholeSymbols,
+    retainedBytes,
     symbolTableSize,
     maxRows,
     totalRows,
@@ -2300,7 +2301,8 @@ function checkMemory({
   bytesHeld = null,
   readBytes = null,
   measured = null,
-  wholeSymbols = false
+  wholeSymbols = false,
+  retainedBytes = 0
 }) {
   if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
     throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
@@ -2324,7 +2326,12 @@ function checkMemory({
     rowsLive,
     wholeSymbols
   );
-  const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
+  const {
+    held,
+    afterRelease: heldAfterRelease = null,
+    forRows: heldForRows,
+    forChunk: heldForChunk
+  } = bytesHeld ?? noBytesHeld;
   const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
   const bounded = budget.candidates.map((candidate) => {
     const heapOnly = candidate.source === "V8 heap limit";
@@ -2360,6 +2367,25 @@ function checkMemory({
   const availableMemory = binding ? binding.bytes : budget.bytes;
   const estimatedMemory = binding ? binding.needs : heapMemory;
   const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
+  const retainedIsTheReason = (() => {
+    if (binding === null || retainedBytes <= 0) {
+      return false;
+    }
+    const heapAfter = estimateMemoryUsage(
+      Math.max(0, symbolTableSize - retainedBytes),
+      maxRows,
+      totalRows,
+      columnCount,
+      materialisesRows,
+      rowsLive,
+      wholeSymbols
+    );
+    const externalAfter = estimateExternalMemory(liveRows, columnCount) + (heldAfterRelease ?? held);
+    return budget.candidates.every((candidate) => {
+      const heapOnly = candidate.source === "V8 heap limit";
+      return (heapOnly ? heapAfter : heapAfter + externalAfter) <= candidate.bytes * safetyFactor;
+    });
+  })();
   if (binding) {
     const includeExternal = !binding.heapOnly;
     const rowsHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForRows(rows) : 0), "rowsHeldBudget");
@@ -2406,13 +2432,17 @@ function checkMemory({
     const knob = chunked ? "chunkSize" : "limit";
     const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
     const nothingFits = recommendedValue === 0;
+    const held2 = retainedIsTheReason;
+    const fixedCost = held2 ? "the columns this file is holding exceed" : "the symbol table alone exceeds";
+    const release = held2 ? `Close the file and open it again to release them, or raise ` : `Raise `;
+    const releaseFirst = held2 ? `Close the file and open it again to release the columns it is holding, which is the direct remedy. ` : "";
     let advice;
     if (nothingFits) {
-      advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
+      advice = `No row count fits this budget - ${fixedCost} it, so ${knob} cannot help. ` + (containerBound ? `${release}the container's memory limit.` : `${release}the heap with --max-old-space-size, or raise memorySafetyFactor.`);
     } else if (containerBound) {
-      advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
+      advice = releaseFirst + `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
     } else {
-      advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
+      advice = releaseFirst + `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
     }
     const suggestions = [];
     if (!nothingFits) {
@@ -2436,10 +2466,16 @@ function checkMemory({
       exact: symbolTableSize === 0,
       suggestions,
       refusal: {
-        message: `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
+        message: `Insufficient memory to load file safely. ${retainedIsTheReason ? "Columns held" : "Symbol table"}: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
         context: {
           symbolTableSize,
           symbolTableSizeMB: sizeMB,
+          // Whether that figure is the file's symbol table or what an open file is still holding. A
+          // paging read is charged for every column any of its pages decoded and kept, so a caller
+          // branching on the refusal needs to know which of the two it is looking at - the remedies
+          // differ, and for this one releasing is a remedy where raising the limit is only a workaround.
+          holdsDecodedColumns: retainedIsTheReason,
+          retainedSymbolBytes: retainedBytes,
           estimatedMemoryMB: estimatedMB,
           availableMemoryMB: availableMB,
           heapLimitMB,
@@ -2607,19 +2643,39 @@ function validateHeaderStructure(headerObj, filePath, stage) {
   });
   return fieldList;
 }
-function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
+function validateSymbolTableSizeEarly(symbolTableLength, filePath, retainedBytes = 0) {
   const heapLimit = getHeapLimit();
   const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
   if (symbolTableLength > MAX_SYMBOL_TABLE_SIZE) {
     const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
     const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
     const heapMB = Math.round(heapLimit / 1024 / 1024);
+    if (retainedBytes > 0 && symbolTableLength - retainedBytes <= MAX_SYMBOL_TABLE_SIZE) {
+      throw new exports.QvdValidationError(
+        `Columns held too large (${sizeMB}MB exceeds ${maxMB}MB limit). This open file is holding the columns its pages have decoded, and they have grown past the ceiling rather than the file's own symbol table being large. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) closing the file and opening it again, which releases what the pages decoded, (2) paging over fewer columns with fields, or (3) increasing heap size with --max-old-space-size.`,
+        {
+          file: filePath,
+          symbolTableSize: symbolTableLength,
+          symbolTableSizeMB: sizeMB,
+          holdsDecodedColumns: true,
+          retainedSymbolBytes: retainedBytes,
+          maxAllowed: MAX_SYMBOL_TABLE_SIZE,
+          maxAllowedMB: maxMB,
+          heapLimitMB: heapMB,
+          reason: "memory"
+        }
+      );
+    }
     throw new exports.QvdValidationError(
       `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
       {
         file: filePath,
         symbolTableSize: symbolTableLength,
         symbolTableSizeMB: sizeMB,
+        // Present and false, not absent. A caller told it can branch on this has to find it on both
+        // refusals, or the branch reads `undefined` for the commoner of the two.
+        holdsDecodedColumns: false,
+        retainedSymbolBytes: retainedBytes,
         maxAllowed: MAX_SYMBOL_TABLE_SIZE,
         maxAllowedMB: maxMB,
         heapLimitMB: heapMB,
@@ -4025,6 +4081,10 @@ var init_QvdFileReader = __esm({
         const columnCount = selected.length;
         const symbolBytes = symbolBytesOf(this._fieldsHeldAfter(selected, headerFields), symbolTableLength);
         const readSymbolBytes = symbolBytesOf(this._fieldsReadBy(selected), symbolTableLength);
+        const retainedBytes = symbolBytesOf(
+          this._fieldsHeldAfter(selected, headerFields).slice(selected.length),
+          symbolTableLength
+        );
         const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
         const windowRows = resolved.limit;
         if (headerNumbersUsable && this._headerMatchesFile) {
@@ -4042,7 +4102,8 @@ var init_QvdFileReader = __esm({
               windowRows,
               recordSize,
               liveRows,
-              this._analysisAhead(window, resolved, totalRows, symbolTableLength)
+              this._analysisAhead(window, resolved, totalRows, symbolTableLength),
+              symbolBytes - retainedBytes
             ),
             // What it reads, which is not what it holds - the records go through one buffer and are not
             // kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
@@ -4054,7 +4115,8 @@ var init_QvdFileReader = __esm({
             // A paging read keeps whole columns, so the estimate must not discount its symbols as a window's
             // sample of them - see `estimateMemoryUsage`. Under-charging is the direction that ends in a
             // heap-limit abort rather than an error.
-            this._symbolCache !== null
+            this._symbolCache !== null,
+            retainedBytes
           );
         }
         if (window.offset === 0 && window.limit === null) {
@@ -4062,7 +4124,7 @@ var init_QvdFileReader = __esm({
           return;
         }
         const rowsToLoad = windowRows;
-        validateSymbolTableSizeEarly(symbolBytes, this._path);
+        validateSymbolTableSizeEarly(symbolBytes, this._path, retainedBytes);
         validateRecordSize(recordSize, this._path, "readData");
         validateRecordCount(totalRows, this._path, "readData");
         const fileBytesRequired = indexTableOffset + (resolved.offset + rowsToLoad) * recordSize;
@@ -4174,9 +4236,14 @@ var init_QvdFileReader = __esm({
        *   would hold.
        * @private
        */
-      _bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead) {
+      _bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead, freshSymbolBytes = null) {
         return {
           held: this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
+          // What it would hold having closed the file and opened it again: every selected column read fresh,
+          // because nothing is cached any more. Higher than `held`, not lower - a cached column this read
+          // selects costs nothing to read now and would cost its area then. Without it the counterfactual
+          // that decides whether releasing helps was answered against the warm figure and said yes too often.
+          afterRelease: freshSymbolBytes === null ? null : this._bytesHeldBy(freshSymbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
           // A window of so many rows reads so many records at a time, and the pass that reads it ahead of the
           // decode reads the same rows, so the buffer is sized from the rows either way.
           forRows: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, rows, recordSize), "forRows"),
@@ -4802,7 +4869,8 @@ var init_QvdFileReader = __esm({
         const symbolTableSize = this._symbolTableLength();
         const plan = this._symbolAreaPlan();
         const readSymbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
-        const symbolBytes = readSymbolBytes + symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
+        const retainedBytes = symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
+        const symbolBytes = readSymbolBytes + retainedBytes;
         const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
         const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
         validateSymbolTableSize(symbolBytes, this._path, totalRows);
@@ -4816,11 +4884,12 @@ var init_QvdFileReader = __esm({
             fields.length,
             this._materialisesRows,
             liveRows,
-            this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false),
+            this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false, symbolBytes - retainedBytes),
             // `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
             // run has read the window's records once already - so the read's total is two passes over them.
             readSymbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize,
-            this._symbolCache !== null
+            this._symbolCache !== null,
+            retainedBytes
           );
         }
         warnLargeSymbolTable(
@@ -5072,12 +5141,14 @@ var init_QvdFileReader = __esm({
        * viewer scrolls to - and the header is already in hand. `parseHeaderOnly` has to have run.
        *
        * @param {QvdRowWindow} window The rows the read would cover, normalised.
-       * @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean}} [options]
-       *   `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask about a read other than the
-       *   one this reader was built for, which is what a `QvdFile` does per call.
+       * @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean,
+       *   paging?: boolean}} [options] `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask
+       *   about a read other than the one this reader was built for, which is what a `QvdFile` does per call;
+       *   `paging` to say whether the read being asked about is a paging read, which defaults to whether this
+       *   reader is one.
        * @return {any} The answer - see `checkMemory`.
        */
-      checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0 } = {}) {
+      checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0, paging = this._symbolCache !== null } = {}) {
         assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
         const header = this._header["QvdTableHeader"];
         const totalRows = headerInteger(header["NoOfRecords"]);
@@ -5088,11 +5159,13 @@ var init_QvdFileReader = __esm({
         const resolved = resolveWindow(window, totalRows);
         const windowRows = resolved.limit;
         const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
-        const analysisAhead = this._analysisAhead(window, resolved, totalRows, symbolTableLength);
+        const analysisAhead = paging ? false : this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
         const measured = getMemoryBudget();
         const ask = /* @__PURE__ */ __name((asked, rows) => {
-          const bytes = symbolBytesOf(this._fieldsHeldAfter(asked, this._allFields), symbolTableLength);
-          const read = symbolBytesOf(this._fieldsReadBy(asked), symbolTableLength);
+          const held = paging ? this._fieldsHeldAfter(asked, this._allFields) : asked;
+          const bytes = symbolBytesOf(held, symbolTableLength);
+          const read = symbolBytesOf(paging ? this._fieldsReadBy(asked) : asked, symbolTableLength);
+          const retained = paging ? symbolBytesOf(held.slice(asked.length), symbolTableLength) : 0;
           return checkMemory({
             measured,
             symbolTableSize: bytes,
@@ -5103,8 +5176,9 @@ var init_QvdFileReader = __esm({
             materialisesRows: builds,
             live: liveRows,
             // A paging read keeps whole columns, so it is charged for whole columns - see `estimateMemoryUsage`.
-            wholeSymbols: this._symbolCache !== null,
-            bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead),
+            wholeSymbols: paging,
+            retainedBytes: retained,
+            bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead, bytes - retained),
             // What it reads from the file, which is not what it holds: the symbol areas it has still to read,
             // and every record the window covers, read a slice at a time and not kept - twice over where the
             // symbol-usage pass will run, since it reads them before the decode reads them again.
@@ -5437,12 +5511,16 @@ var init_QvdFile = __esm({
       /**
        * What a read of this file would cost, and whether it fits - with no I/O at all.
        *
-       * The same answer `QvdDataFrame.checkRead()` gives, from the header this file already holds, so a
-       * viewer can size a page before asking for it without touching the disk.
+       * Without `chunkSize` it asks about a page, and answers for the page this file would read next: the
+       * columns it names plus the ones earlier pages decoded and kept, since those are part of what the page
+       * costs. With `chunkSize` it asks about an `iterate()` of the file instead - which this object cannot
+       * make, but a caller holding it may want to weigh - and answers exactly as `QvdDataFrame.checkRead()`
+       * would, from the header this file already holds rather than by reading it again.
        *
        * @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
        *   as?: 'rows'|'columns', chunkSize?: number|null}} [options] The read being asked about - the same
-       *   bag `rows()` takes, plus `as` and `chunkSize` to say which shape of read it is.
+       *   bag `rows()` takes, plus `as` to say which shape of page, and `chunkSize` to ask about an
+       *   `iterate()` of the file rather than a page.
        * @return {any} The answer - `fits`, `reason`, `estimate`, `budget`, `exact`, `suggestions`.
        * @throws {QvdValidationError} If the file is closed, or an option's value is not valid.
        */
@@ -5461,7 +5539,7 @@ var init_QvdFile = __esm({
         if (chunkSize !== null) {
           requireChunkSize(chunkSize, this._options.path);
         }
-        const reader = this._readers[as] ?? this._reader;
+        const reader = chunkSize !== null ? this._reader : this._readers[as] ?? this._reader;
         return reader.checkParsed(normaliseWindow(windowFrom(options), this._options.path), {
           chunkSize,
           // Resolved here rather than left to the reader, exactly as `_page` resolves it. A warm reader is
@@ -5470,7 +5548,12 @@ var init_QvdFile = __esm({
           // file after a page naming one of them, it reported 27,490 bytes for a read that costs 108,160:
           // understating, which is the direction that approves a read the read then refuses.
           fields: options.fields === void 0 ? this._options.fields ?? null : options.fields,
-          materialisesRows: as === "rows"
+          materialisesRows: as === "rows",
+          // Said outright rather than read off the reader. The header reader has paging on, because a check
+          // made before the first page has to be priced as that page will be - so it cannot tell an `iterate()`
+          // question from a page one by its own mode, and routing to it alone still priced the iterate as a
+          // page wherever a bounded window would have been filtered.
+          ...chunkSize === null ? {} : { paging: false }
         });
       }
       /**