qvdjs 0.9.2 → 0.9.3

package/dist/index.js

@@ -734,6 +734,28 @@ var init_bitUtils = __esm({
 function getHeapLimit() {
   return v8.getHeapStatistics().heap_size_limit;
 }
+function heapLimitIsMeaningful() {
+  return !process.versions.bun && !process.versions.deno;
+}
+function getMemoryBudget() {
+  const candidates = [];
+  if (heapLimitIsMeaningful()) {
+    candidates.push({ source: "V8 heap limit", bytes: getHeapLimit() });
+  }
+  const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
+  if (constrained > 0 && constrained < os.totalmem()) {
+    candidates.push({ source: "container memory limit", bytes: constrained });
+  }
+  if (candidates.length === 0) {
+    candidates.push({ source: "total system memory", bytes: os.totalmem() });
+  }
+  const observed = [{ source: "free memory (os.freemem)", bytes: os.freemem() }];
+  if (typeof process.availableMemory === "function") {
+    observed.push({ source: "available memory", bytes: process.availableMemory() });
+  }
+  const binding = candidates.reduce((lowest, candidate) => candidate.bytes < lowest.bytes ? candidate : lowest);
+  return { bytes: binding.bytes, limitedBy: binding.source, candidates, observed };
+}
 function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
   const FULL_PARSE_OVERHEAD = 6;
   const MINIMAL_OVERHEAD = 0.01;
@@ -750,11 +772,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
   if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
     throw new QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
   }
-  const availableMemory = os.freemem();
+  if (safetyFactor === 0) {
+    return;
+  }
+  const budget = getMemoryBudget();
   const heapLimit = getHeapLimit();
+  const availableMemory = budget.bytes;
   const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
-  const effectiveLimit = Math.min(availableMemory, heapLimit);
-  const maxAllowedMemory = effectiveLimit * safetyFactor;
+  const maxAllowedMemory = budget.bytes * safetyFactor;
   if (estimatedMemory > maxAllowedMemory) {
     const safeSymbolPercentage = maxAllowedMemory / (symbolTableSize * 6);
     const safeRowPercentage = Math.pow(safeSymbolPercentage, 2);
@@ -764,9 +789,12 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
     const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
     const heapLimitMB = Math.round(heapLimit / 1024 / 1024);
     const availableRamMB = Math.round(availableMemory / 1024 / 1024);
-    const limitingFactor = heapLimit < availableMemory ? "V8 heap limit" : "available RAM";
+    const limitingFactor = budget.limitedBy;
+    const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
+    const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
+    const advice = limitingFactor === "container memory limit" ? `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).` : `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
     throw new QvdValidationError(
-      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}: ${heapLimitMB}MB heap / ${availableRamMB}MB RAM). Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`,
+      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
       {
         file: filePath,
         symbolTableSize,
@@ -776,6 +804,8 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
         heapLimitMB,
         availableRamMB,
         limitingFactor,
+        memoryBudget: budget.candidates,
+        memoryObserved: budget.observed,
         totalRows,
         maxRows,
         recommendedMaxRows
@@ -1289,14 +1319,22 @@ var init_QvdFileReader = __esm({
        *   points outside it is rejected. Defaults to the current working directory. To permit
        *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
        *   empty value falls back to the working directory rather than removing the restriction.
-       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
-       *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
-       *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
+       * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a
+       *   load may use. The budget is the smallest of the V8 heap limit, any container memory limit,
+       *   and the memory the OS reports as available. Default is 0.3. **Zero disables the memory
+       *   check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
+       *   Bun reports its current heap as its heap limit - and for callers who would rather manage
+       *   memory themselves than trust the estimate.
+       * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes,
+       *   above which a lazy load switches to the two-pass filtering path. The default of 50MB is
+       *   the point where the extra analysis pass pays for itself; lower it to use filtering on
+       *   smaller files, raise it to keep the simpler single-pass read for longer.
        */
       constructor(filePath, options = {}) {
-        const { allowedDir, memorySafetyFactor = 0.3 } = options;
+        const { allowedDir, memorySafetyFactor = 0.3, symbolFilteringThreshold = 50 * 1024 * 1024 } = options;
         this._path = validatePath(filePath, allowedDir);
         this._memorySafetyFactor = memorySafetyFactor;
+        this._symbolFilteringThreshold = symbolFilteringThreshold;
         this._buffer = null;
         this._headerOffset = null;
         this._symbolTableOffset = null;
@@ -1728,15 +1766,12 @@ var init_QvdFileReader = __esm({
         await this._readData(maxRows);
         await this._parseHeader();
         let symbolsToKeep = null;
+        let symbolsKept = null;
         if (maxRows !== null && this._header) {
           const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
-          const SYMBOL_FILTERING_THRESHOLD = 50 * 1024 * 1024;
-          if (symbolTableLength > SYMBOL_FILTERING_THRESHOLD) {
+          if (symbolTableLength > this._symbolFilteringThreshold) {
             symbolsToKeep = await this._analyzeIndexTableSymbolUsage(maxRows);
-            const totalSymbols = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
-            console.log(
-              `[Phase 2.5 Optimization] Using stream-and-skip parsing: keeping ${totalSymbols} symbols from ${(symbolTableLength / 1024 / 1024).toFixed(1)}MB symbol table`
-            );
+            symbolsKept = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
           }
         }
         await this._parseSymbolTable(symbolsToKeep, maxRows);
@@ -1773,7 +1808,14 @@ var init_QvdFileReader = __esm({
         const columns = fields.map((field) => field["FieldName"]);
         const data = this._indexTable.map((_, index) => getRow(index));
         const metadata = this._header["QvdTableHeader"];
-        return new QvdDataFrame(data, columns, metadata);
+        const loadStats = {
+          symbolTableBytes: parseInt(this._header["QvdTableHeader"]["Offset"], 10),
+          totalRows: parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10),
+          rowsLoaded: data.length,
+          symbolFiltering: symbolsToKeep !== null,
+          symbolsKept
+        };
+        return new QvdDataFrame(data, columns, metadata, loadStats);
       }
     };
   }
@@ -1790,11 +1832,13 @@ var init_QvdDataFrame = __esm({
        * @param {Array<Array<any>>} data The data of the data frame.
        * @param {Array<string>} columns The columns of the data frame.
        * @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
+       * @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
        */
-      constructor(data, columns, metadata = null) {
+      constructor(data, columns, metadata = null, loadStats = null) {
         this._data = data;
         this._columns = columns;
         this._metadata = metadata;
+        this._loadStats = loadStats;
       }
       /**
        * Returns the data of the data frame.
@@ -1821,6 +1865,21 @@ var init_QvdDataFrame = __esm({
       get metadata() {
         return this._metadata;
       }
+      /**
+       * Returns statistics about the read that produced this data frame.
+       *
+       * Only a frame returned by fromQvd() carries these; fromDict(), head() and tail() produce
+       * frames that describe no particular read, and report null rather than a stale figure.
+       *
+       * The main use is confirming that a lazy load actually filtered the symbol table:
+       * `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
+       * survived it, which for a small maxRows should be a tiny fraction of the file's total.
+       *
+       * @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
+       */
+      get loadStats() {
+        return this._loadStats;
+      }
       /**
        * Returns file-level metadata from the QVD header.
        * @return {Object} File-level metadata properties.
@@ -2196,9 +2255,12 @@ var init_QvdDataFrame = __esm({
        *   outside it is rejected. Defaults to the current working directory. To permit an entire
        *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
        *   back to the working directory rather than removing the restriction.
-       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
-       *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
-       *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
+       * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a load
+       *   may use. The budget is the smallest of the V8 heap limit, any container memory limit, and the
+       *   memory the OS reports as available. Default is 0.3; raise it when running with a larger heap via
+       *   --max-old-space-size. **Zero disables the memory check entirely.**
+       * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
+       *   a lazy load switches to the two-pass filtering path. Defaults to 50MB.
        * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
@@ -2206,7 +2268,8 @@ var init_QvdDataFrame = __esm({
         const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
         const readerOptions = {
           allowedDir: options.allowedDir,
-          memorySafetyFactor: options.memorySafetyFactor
+          memorySafetyFactor: options.memorySafetyFactor,
+          symbolFilteringThreshold: options.symbolFilteringThreshold
         };
         return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
       }