qvdjs 0.7.0 → 0.8.0

package/dist/index.js

@@ -4,6 +4,7 @@ import crypto from 'crypto';
 import xml2 from 'xml2js';
 import assert from 'assert';
 import os from 'os';
+import v8 from 'v8';
 
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
@@ -647,6 +648,9 @@ var init_bitUtils = __esm({
   "src/util/bitUtils.js"() {
   }
 });
+function getHeapLimit() {
+  return v8.getHeapStatistics().heap_size_limit;
+}
 function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
   const FULL_PARSE_OVERHEAD = 6;
   const MINIMAL_OVERHEAD = 0.01;
@@ -659,11 +663,12 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
   const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
   return keptSymbolsMemory + skippedSymbolsMemory;
 }
-function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath) {
+function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.3) {
   const availableMemory = os.freemem();
+  const heapLimit = getHeapLimit();
   const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
-  const MEMORY_SAFETY_FACTOR = 0.3;
-  const maxAllowedMemory = availableMemory * MEMORY_SAFETY_FACTOR;
+  const effectiveLimit = Math.min(availableMemory, heapLimit);
+  const maxAllowedMemory = effectiveLimit * safetyFactor;
   if (estimatedMemory > maxAllowedMemory) {
     const safeSymbolPercentage = maxAllowedMemory / (symbolTableSize * 6);
     const safeRowPercentage = Math.pow(safeSymbolPercentage, 2);
@@ -671,14 +676,20 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
     const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
     const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
     const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
+    const heapLimitMB = Math.round(heapLimit / 1024 / 1024);
+    const availableRamMB = Math.round(availableMemory / 1024 / 1024);
+    const limitingFactor = heapLimit < availableMemory ? "V8 heap limit" : "available RAM";
     throw new QvdValidationError(
-      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB. Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`,
+      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}: ${heapLimitMB}MB heap / ${availableRamMB}MB RAM). Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`,
       {
         file: filePath,
         symbolTableSize,
         symbolTableSizeMB: sizeMB,
         estimatedMemoryMB: estimatedMB,
         availableMemoryMB: availableMB,
+        heapLimitMB,
+        availableRamMB,
+        limitingFactor,
         totalRows,
         maxRows,
         recommendedMaxRows
@@ -687,13 +698,15 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
   }
 }
 function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows) {
-  const LARGE_SYMBOL_TABLE_WARNING = 500 * 1024 * 1024;
+  const heapLimit = getHeapLimit();
+  const LARGE_SYMBOL_TABLE_WARNING = heapLimit * 0.125;
   if (symbolTableSize > LARGE_SYMBOL_TABLE_WARNING && (maxRows === null || maxRows >= totalRows)) {
     const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
     const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
     const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
+    const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
     console.warn(
-      `\u26A0\uFE0F  Large symbol table detected (${sizeMB}MB). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
+      `\u26A0\uFE0F  Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
     );
   }
 }
@@ -705,35 +718,43 @@ var init_memoryUtils = __esm({
 
 // src/util/validationUtils.js
 function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
-  const MAX_SYMBOL_TABLE_SIZE = 500 * 1024 * 1024;
+  const heapLimit = getHeapLimit();
+  const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
   if (symbolTableLength > MAX_SYMBOL_TABLE_SIZE) {
     const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
     const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
+    const heapMB = Math.round(heapLimit / 1024 / 1024);
     throw new QvdValidationError(
-      `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit). This QVD file contains extremely high-cardinality fields. Even with symbol filtering, this file cannot be loaded efficiently. Consider: (1) using a subset of the data, (2) aggregating high-cardinality fields, or (3) using a different data format.`,
+      `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
       {
         file: filePath,
         symbolTableSize: symbolTableLength,
         symbolTableSizeMB: sizeMB,
         maxAllowed: MAX_SYMBOL_TABLE_SIZE,
-        maxAllowedMB: maxMB
+        maxAllowedMB: maxMB,
+        heapLimitMB: heapMB,
+        limitPercentage: 12.5
       }
     );
   }
 }
 function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
-  const ABSOLUTE_MAX_SYMBOL_TABLE = 2 * 1024 * 1024 * 1024;
+  const heapLimit = getHeapLimit();
+  const ABSOLUTE_MAX_SYMBOL_TABLE = heapLimit * 0.5;
   if (symbolTableLength > ABSOLUTE_MAX_SYMBOL_TABLE) {
     const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
     const maxMB = Math.round(ABSOLUTE_MAX_SYMBOL_TABLE / 1024 / 1024);
+    const heapMB = Math.round(heapLimit / 1024 / 1024);
     throw new QvdValidationError(
-      `Symbol table exceeds absolute maximum size (${sizeMB}MB > ${maxMB}MB). This QVD file has pathological cardinality (likely a data modeling issue). Even with Phase 2.5 optimization, files this large cannot be processed safely. Consider: (1) aggregating high-cardinality fields, (2) splitting the data, or (3) using a different format.`,
+      `Symbol table exceeds absolute maximum size (${sizeMB}MB > ${maxMB}MB). This QVD file has pathological cardinality (likely a data modeling issue). Limit scales with heap size (current: ${heapMB}MB, limit: 50% = ${maxMB}MB). Consider: (1) increasing heap size with --max-old-space-size, (2) aggregating high-cardinality fields, (3) splitting the data, or (4) using a different format.`,
       {
         file: filePath,
         symbolTableSize: symbolTableLength,
         symbolTableSizeMB: sizeMB,
         maxAllowed: ABSOLUTE_MAX_SYMBOL_TABLE,
         maxAllowedMB: maxMB,
+        heapLimitMB: heapMB,
+        limitPercentage: 50,
         totalRows
       }
     );
@@ -870,6 +891,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
 var init_validationUtils = __esm({
   "src/util/validationUtils.js"() {
     init_QvdErrors();
+    init_memoryUtils();
   }
 });
 
@@ -1152,10 +1174,14 @@ var init_QvdFileReader = __esm({
        * @param {Object} [options={}] Options for the reader.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
        *   path must be within this directory. Defaults to current working directory.
+       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
+       *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
+       *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
        */
       constructor(filePath, options = {}) {
-        const { allowedDir } = options;
+        const { allowedDir, memorySafetyFactor = 0.3 } = options;
         this._path = validatePath(filePath, allowedDir);
+        this._memorySafetyFactor = memorySafetyFactor;
         this._buffer = null;
         this._headerOffset = null;
         this._symbolTableOffset = null;
@@ -1379,7 +1405,7 @@ var init_QvdFileReader = __esm({
         const symbolTableSize = symbolBuffer.length;
         const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
         validateSymbolTableSize(symbolTableSize, this._path, totalRows);
-        validateMemoryAvailability(symbolTableSize, maxRows, totalRows, this._path);
+        validateMemoryAvailability(symbolTableSize, maxRows, totalRows, this._path, this._memorySafetyFactor);
         warnLargeSymbolTable(symbolTableSize, maxRows, totalRows);
         if (!Array.isArray(fields)) {
           fields = [fields];
@@ -1960,11 +1986,17 @@ var init_QvdDataFrame = __esm({
        * @param {Object} [options] Optional loading options.
        * @param {number|null} [options.maxRows] The maximum number of rows to load. If not specified, all rows are loaded.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
+       *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
+       *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
       static async fromQvd(path3, options = {}) {
         const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
-        const readerOptions = { allowedDir: options.allowedDir };
+        const readerOptions = {
+          allowedDir: options.allowedDir,
+          memorySafetyFactor: options.memorySafetyFactor
+        };
         return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
       }
       /**