qvdjs 0.6.2 → 0.8.0

package/dist/index.js

@@ -3,6 +3,8 @@ import fs2 from 'fs';
 import crypto from 'crypto';
 import xml2 from 'xml2js';
 import assert from 'assert';
+import os from 'os';
+import v8 from 'v8';
 
 var __defProp = Object.defineProperty;
 var __getOwnPropNames = Object.getOwnPropertyNames;
@@ -635,6 +637,520 @@ var init_QvdFileWriter = __esm({
   }
 });
 
+// src/util/bitUtils.js
+function convertBitsToInt32(bits) {
+  if (bits.length === 0) {
+    return 0;
+  }
+  return bits.reduce((value, bit, index) => value += bit * Math.pow(2, index), 0);
+}
+var init_bitUtils = __esm({
+  "src/util/bitUtils.js"() {
+  }
+});
+function getHeapLimit() {
+  return v8.getHeapStatistics().heap_size_limit;
+}
+function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
+  const FULL_PARSE_OVERHEAD = 6;
+  const MINIMAL_OVERHEAD = 0.01;
+  if (maxRows === null || maxRows >= totalRows) {
+    return symbolTableSize * FULL_PARSE_OVERHEAD;
+  }
+  const rowPercentage = maxRows / totalRows;
+  const symbolPercentage = Math.sqrt(rowPercentage);
+  const keptSymbolsMemory = symbolTableSize * symbolPercentage * FULL_PARSE_OVERHEAD;
+  const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
+  return keptSymbolsMemory + skippedSymbolsMemory;
+}
+function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.3) {
+  const availableMemory = os.freemem();
+  const heapLimit = getHeapLimit();
+  const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
+  const effectiveLimit = Math.min(availableMemory, heapLimit);
+  const maxAllowedMemory = effectiveLimit * safetyFactor;
+  if (estimatedMemory > maxAllowedMemory) {
+    const safeSymbolPercentage = maxAllowedMemory / (symbolTableSize * 6);
+    const safeRowPercentage = Math.pow(safeSymbolPercentage, 2);
+    const recommendedMaxRows = Math.max(1, Math.floor(totalRows * safeRowPercentage));
+    const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
+    const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
+    const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
+    const heapLimitMB = Math.round(heapLimit / 1024 / 1024);
+    const availableRamMB = Math.round(availableMemory / 1024 / 1024);
+    const limitingFactor = heapLimit < availableMemory ? "V8 heap limit" : "available RAM";
+    throw new QvdValidationError(
+      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}: ${heapLimitMB}MB heap / ${availableRamMB}MB RAM). Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`,
+      {
+        file: filePath,
+        symbolTableSize,
+        symbolTableSizeMB: sizeMB,
+        estimatedMemoryMB: estimatedMB,
+        availableMemoryMB: availableMB,
+        heapLimitMB,
+        availableRamMB,
+        limitingFactor,
+        totalRows,
+        maxRows,
+        recommendedMaxRows
+      }
+    );
+  }
+}
+function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows) {
+  const heapLimit = getHeapLimit();
+  const LARGE_SYMBOL_TABLE_WARNING = heapLimit * 0.125;
+  if (symbolTableSize > LARGE_SYMBOL_TABLE_WARNING && (maxRows === null || maxRows >= totalRows)) {
+    const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
+    const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
+    const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
+    const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
+    console.warn(
+      `\u26A0\uFE0F  Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
+    );
+  }
+}
+var init_memoryUtils = __esm({
+  "src/util/memoryUtils.js"() {
+    init_QvdErrors();
+  }
+});
+
+// src/util/validationUtils.js
+function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
+  const heapLimit = getHeapLimit();
+  const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
+  if (symbolTableLength > MAX_SYMBOL_TABLE_SIZE) {
+    const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
+    const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
+    const heapMB = Math.round(heapLimit / 1024 / 1024);
+    throw new QvdValidationError(
+      `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
+      {
+        file: filePath,
+        symbolTableSize: symbolTableLength,
+        symbolTableSizeMB: sizeMB,
+        maxAllowed: MAX_SYMBOL_TABLE_SIZE,
+        maxAllowedMB: maxMB,
+        heapLimitMB: heapMB,
+        limitPercentage: 12.5
+      }
+    );
+  }
+}
+function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
+  const heapLimit = getHeapLimit();
+  const ABSOLUTE_MAX_SYMBOL_TABLE = heapLimit * 0.5;
+  if (symbolTableLength > ABSOLUTE_MAX_SYMBOL_TABLE) {
+    const sizeMB = Math.round(symbolTableLength / 1024 / 1024);
+    const maxMB = Math.round(ABSOLUTE_MAX_SYMBOL_TABLE / 1024 / 1024);
+    const heapMB = Math.round(heapLimit / 1024 / 1024);
+    throw new QvdValidationError(
+      `Symbol table exceeds absolute maximum size (${sizeMB}MB > ${maxMB}MB). This QVD file has pathological cardinality (likely a data modeling issue). Limit scales with heap size (current: ${heapMB}MB, limit: 50% = ${maxMB}MB). Consider: (1) increasing heap size with --max-old-space-size, (2) aggregating high-cardinality fields, (3) splitting the data, or (4) using a different format.`,
+      {
+        file: filePath,
+        symbolTableSize: symbolTableLength,
+        symbolTableSizeMB: sizeMB,
+        maxAllowed: ABSOLUTE_MAX_SYMBOL_TABLE,
+        maxAllowedMB: maxMB,
+        heapLimitMB: heapMB,
+        limitPercentage: 50,
+        totalRows
+      }
+    );
+  }
+}
+function validateFieldMetadata(field, symbolBufferLength, filePath) {
+  const symbolsOffset = parseInt(field["Offset"], 10);
+  const symbolsLength = parseInt(field["Length"], 10);
+  if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
+    throw new QvdCorruptedError("Invalid symbol offset", {
+      field: field["FieldName"],
+      offset: symbolsOffset,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  if (isNaN(symbolsLength) || !Number.isSafeInteger(symbolsLength) || symbolsLength < 0) {
+    throw new QvdCorruptedError("Invalid symbol length", {
+      field: field["FieldName"],
+      length: symbolsLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  if (symbolsOffset + symbolsLength > symbolBufferLength) {
+    throw new QvdCorruptedError("Symbol data extends beyond buffer", {
+      field: field["FieldName"],
+      offset: symbolsOffset,
+      length: symbolsLength,
+      bufferSize: symbolBufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+}
+function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath) {
+  if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
+    throw new QvdCorruptedError("Invalid record byte size", {
+      recordSize,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
+    throw new QvdCorruptedError("Invalid number of records", {
+      totalRows,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (recordSize === 0 && totalRows > 0) {
+    throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
+      recordSize,
+      totalRows,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (recordSize > 1048576) {
+    throw new QvdCorruptedError("Record byte size exceeds maximum", {
+      recordSize,
+      maxSize: 1048576,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (isNaN(indexTableLength) || !Number.isSafeInteger(indexTableLength) || indexTableLength < 0) {
+    throw new QvdCorruptedError("Invalid index table length", {
+      length: indexTableLength,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (indexTableOffset > bufferLength) {
+    throw new QvdCorruptedError("Index table offset beyond buffer", {
+      indexTableOffset,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  const maxReasonableOverage = 100 * 1024 * 1024;
+  if (indexTableLength > bufferLength + maxReasonableOverage) {
+    throw new QvdCorruptedError("Index table length unreasonably large", {
+      indexTableLength,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  const requiredIndexBytes = rowsToLoad * recordSize;
+  if (indexTableLength < requiredIndexBytes) {
+    throw new QvdCorruptedError("Index table length smaller than required", {
+      indexTableLength,
+      requiredBytes: requiredIndexBytes,
+      rowsToLoad,
+      recordSize,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+}
+function validateFieldBitMetadata(field, recordSize, filePath) {
+  const bitOffset = parseInt(field["BitOffset"], 10);
+  const bitWidth = parseInt(field["BitWidth"], 10);
+  if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
+    throw new QvdCorruptedError("Invalid bit offset", {
+      field: field["FieldName"],
+      bitOffset,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (isNaN(bitWidth) || !Number.isSafeInteger(bitWidth) || bitWidth < 0) {
+    throw new QvdCorruptedError("Invalid bit width", {
+      field: field["FieldName"],
+      bitWidth,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  const recordSizeInBits = recordSize * 8;
+  if (bitOffset + bitWidth > recordSizeInBits) {
+    throw new QvdCorruptedError("Bit field extends beyond record size", {
+      field: field["FieldName"],
+      bitOffset,
+      bitWidth,
+      recordSizeInBits,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+}
+var init_validationUtils = __esm({
+  "src/util/validationUtils.js"() {
+    init_QvdErrors();
+    init_memoryUtils();
+  }
+});
+
+// src/util/symbolParser.js
+function parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  if (pointer + 4 > bufferLength) {
+    throw new QvdCorruptedError("Buffer overflow reading integer symbol", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
+  const value = Buffer.from(byteData).readIntLE(0, byteData.length);
+  return { symbol: QvdSymbol.fromIntValue(value), bytesRead: 4 };
+}
+function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  if (pointer + 8 > bufferLength) {
+    throw new QvdCorruptedError("Buffer overflow reading double symbol", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
+  const value = Buffer.from(byteData).readDoubleLE(0);
+  return { symbol: QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
+}
+function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  const startPointer = pointer;
+  const maxStringLength = 1048576;
+  let stringLength = 0;
+  while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
+    if (stringLength >= maxStringLength) {
+      throw new QvdCorruptedError("String symbol exceeds maximum length", {
+        field: fieldName,
+        maxLength: maxStringLength,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+    pointer++;
+    stringLength++;
+  }
+  if (pointer >= bufferLength) {
+    throw new QvdCorruptedError("String symbol not null-terminated", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const value = symbolBuffer.subarray(startPointer, pointer).toString("utf-8");
+  return { symbol: QvdSymbol.fromStringValue(value), bytesRead: pointer - startPointer + 1 };
+}
+function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  const startPointer = pointer;
+  const maxStringLength = 1048576;
+  let stringLength = 0;
+  while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
+    if (stringLength >= maxStringLength) {
+      throw new QvdCorruptedError("String symbol exceeds maximum length", {
+        field: fieldName,
+        maxLength: maxStringLength,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+    pointer++;
+    stringLength++;
+  }
+  if (pointer >= bufferLength) {
+    throw new QvdCorruptedError("String symbol not null-terminated", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  return pointer - startPointer + 1;
+}
+function parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  if (pointer + 4 > bufferLength) {
+    throw new QvdCorruptedError("Buffer overflow reading dual integer symbol", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
+  const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
+  pointer += 4;
+  const stringStart = pointer;
+  const maxStringLength = 1048576;
+  let stringLength = 0;
+  while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
+    if (stringLength >= maxStringLength) {
+      throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
+        field: fieldName,
+        maxLength: maxStringLength,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+    pointer++;
+    stringLength++;
+  }
+  if (pointer >= bufferLength) {
+    throw new QvdCorruptedError("Dual string symbol not null-terminated", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
+  return { symbol: QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
+}
+function parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
+  if (pointer + 8 > bufferLength) {
+    throw new QvdCorruptedError("Buffer overflow reading dual double symbol", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
+  const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
+  pointer += 8;
+  const stringStart = pointer;
+  const maxStringLength = 1048576;
+  let stringLength = 0;
+  while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
+    if (stringLength >= maxStringLength) {
+      throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
+        field: fieldName,
+        maxLength: maxStringLength,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+    pointer++;
+    stringLength++;
+  }
+  if (pointer >= bufferLength) {
+    throw new QvdCorruptedError("Dual string symbol not null-terminated", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
+  return {
+    symbol: QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
+    bytesRead: pointer - (stringStart - 8) + 1
+  };
+}
+function skipDualSymbol(symbolBuffer, pointer, bufferLength, numericBytes, fieldName, filePath) {
+  if (pointer + numericBytes > bufferLength) {
+    throw new QvdCorruptedError(`Buffer overflow reading dual symbol`, {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  const startPointer = pointer;
+  pointer += numericBytes;
+  const maxStringLength = 1048576;
+  let stringLength = 0;
+  while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
+    if (stringLength >= maxStringLength) {
+      throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
+        field: fieldName,
+        maxLength: maxStringLength,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+    pointer++;
+    stringLength++;
+  }
+  if (pointer >= bufferLength) {
+    throw new QvdCorruptedError("Dual string symbol not null-terminated", {
+      field: fieldName,
+      pointer,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+  return pointer - startPointer + 1;
+}
+function parseSymbol(typeByte, symbolBuffer, pointer, bufferLength, fieldName, filePath, shouldParse) {
+  switch (typeByte) {
+    case 1: {
+      if (shouldParse) {
+        return parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      }
+      return { symbol: null, bytesRead: 4 };
+    }
+    case 2: {
+      if (shouldParse) {
+        return parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      }
+      return { symbol: null, bytesRead: 8 };
+    }
+    case 4: {
+      if (shouldParse) {
+        return parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      }
+      const bytesRead = skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      return { symbol: null, bytesRead };
+    }
+    case 5: {
+      if (shouldParse) {
+        return parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      }
+      const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 4, fieldName, filePath);
+      return { symbol: null, bytesRead };
+    }
+    case 6: {
+      if (shouldParse) {
+        return parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
+      }
+      const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
+      return { symbol: null, bytesRead };
+    }
+    default: {
+      throw new QvdParseError("Unknown symbol type byte", {
+        typeByte: typeByte.toString(16),
+        offset: pointer - 1,
+        file: filePath,
+        stage: "parseSymbolTable"
+      });
+    }
+  }
+}
+var init_symbolParser = __esm({
+  "src/util/symbolParser.js"() {
+    init_QvdSymbol();
+    init_QvdErrors();
+  }
+});
+
 // src/QvdFileReader.js
 var QvdFileReader_exports = {};
 __export(QvdFileReader_exports, {
@@ -643,10 +1159,13 @@ __export(QvdFileReader_exports, {
 var QvdFileReader;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
-    init_QvdSymbol();
     init_QvdDataFrame();
     init_QvdErrors();
     init_validatePath();
+    init_bitUtils();
+    init_memoryUtils();
+    init_validationUtils();
+    init_symbolParser();
     QvdFileReader = class {
       /**
        * Constructs a new QVD file parser.
@@ -655,10 +1174,14 @@ var init_QvdFileReader = __esm({
        * @param {Object} [options={}] Options for the reader.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
        *   path must be within this directory. Defaults to current working directory.
+       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
+       *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
+       *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
        */
       constructor(filePath, options = {}) {
-        const { allowedDir } = options;
+        const { allowedDir, memorySafetyFactor = 0.3 } = options;
         this._path = validatePath(filePath, allowedDir);
+        this._memorySafetyFactor = memorySafetyFactor;
         this._buffer = null;
         this._headerOffset = null;
         this._symbolTableOffset = null;
@@ -744,6 +1267,7 @@ var init_QvdFileReader = __esm({
         const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
         const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
         const rowsToLoad = Math.min(maxRows, totalRows);
+        validateSymbolTableSizeEarly(symbolTableLength, this._path);
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
         const fd = await fs2.promises.open(this._path, "r");
@@ -793,11 +1317,80 @@ var init_QvdFileReader = __esm({
         this._symbolTableOffset = headerEndIndex;
         this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
       }
+      /**
+       * Estimates the memory usage for loading a QVD file with given parameters.
+       * Takes into account Phase 2.5 optimization which skips parsing unused symbols.
+       *
+       * @param {number} symbolTableSize Size of symbol table in bytes
+       * @param {number|null} maxRows Maximum number of rows to load (null = all rows)
+       * @param {number} totalRows Total number of rows in the file
+       * @returns {number} Estimated memory usage in bytes
+       * @private
+       */
+      _estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
+        return estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
+      }
+      /**
+       * Analyzes the index table to determine which symbols are actually needed.
+       * This is used for two-pass symbol filtering optimization.
+       *
+       * @param {number} maxRows The maximum number of rows to analyze.
+       * @returns {Promise<Map<string, Set<number>>>} Map of field names to Set of needed symbol indices.
+       * @private
+       */
+      async _analyzeIndexTableSymbolUsage(maxRows) {
+        if (!this._buffer || !this._header || !this._indexTableOffset) {
+          throw new QvdCorruptedError(
+            "The QVD file has not been loaded in the proper order or has not been loaded at all.",
+            {
+              file: this._path,
+              stage: "analyzeIndexTableSymbolUsage"
+            }
+          );
+        }
+        let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
+        if (!Array.isArray(fields)) {
+          fields = [fields];
+        }
+        const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
+        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
+        const rowsToAnalyze = Math.min(maxRows, totalRows);
+        const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
+        const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
+        const symbolUsage = /* @__PURE__ */ new Map();
+        fields.forEach((field) => {
+          symbolUsage.set(field["FieldName"], /* @__PURE__ */ new Set());
+        });
+        for (let pointer = 0, rowCount = 0; pointer < indexBuffer.length && rowCount < rowsToAnalyze; pointer += recordSize, rowCount++) {
+          const bytes = new Int32Array(indexBuffer.subarray(pointer, pointer + recordSize));
+          bytes.reverse();
+          const mask = bytes.reduce((bits, byte) => bits + ("00000000" + byte.toString(2)).slice(-8), "").split("").reverse().map((bit) => parseInt(bit));
+          fields.forEach((field) => {
+            const bitOffset = parseInt(field["BitOffset"], 10);
+            const bitWidth = parseInt(field["BitWidth"], 10);
+            const bias = parseInt(field["Bias"], 10);
+            let symbolIndex;
+            if (bitWidth === 0) {
+              symbolIndex = 0;
+            } else {
+              symbolIndex = this._convertBitsToInt32(mask.slice(bitOffset, bitOffset + bitWidth));
+            }
+            symbolIndex -= bias;
+            const fieldName = field["FieldName"];
+            symbolUsage.get(fieldName)?.add(symbolIndex);
+          });
+        }
+        return symbolUsage;
+      }
       /**
        * Parses the symbol table of the QVD file. This method is part of the parsing process
        * and should not be called directly.
+       *
+       * @param {Map<string, Set<number>>|null} symbolsToKeep Optional map of field names to symbol indices to keep.
+       *   If provided, only these symbols will be parsed (two-pass filtering optimization).
+       * @param {number|null} maxRows Optional maximum number of rows being loaded (for memory estimation).
        */
-      async _parseSymbolTable() {
+      async _parseSymbolTable(symbolsToKeep = null, maxRows = null) {
         if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset) {
           throw new QvdCorruptedError(
             "The QVD file has not been loaded in the proper order or has not been loaded at all.",
@@ -809,192 +1402,40 @@ var init_QvdFileReader = __esm({
         }
         let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
         const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
+        const symbolTableSize = symbolBuffer.length;
+        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
+        validateSymbolTableSize(symbolTableSize, this._path, totalRows);
+        validateMemoryAvailability(symbolTableSize, maxRows, totalRows, this._path, this._memorySafetyFactor);
+        warnLargeSymbolTable(symbolTableSize, maxRows, totalRows);
         if (!Array.isArray(fields)) {
           fields = [fields];
         }
         for (const field of fields) {
-          const symbolsOffset = parseInt(field["Offset"], 10);
-          const symbolsLength = parseInt(field["Length"], 10);
-          if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
-            throw new QvdCorruptedError("Invalid symbol offset", {
-              field: field["FieldName"],
-              offset: symbolsOffset,
-              file: this._path,
-              stage: "parseSymbolTable"
-            });
-          }
-          if (isNaN(symbolsLength) || !Number.isSafeInteger(symbolsLength) || symbolsLength < 0) {
-            throw new QvdCorruptedError("Invalid symbol length", {
-              field: field["FieldName"],
-              length: symbolsLength,
-              file: this._path,
-              stage: "parseSymbolTable"
-            });
-          }
-          if (symbolsOffset + symbolsLength > symbolBuffer.length) {
-            throw new QvdCorruptedError("Symbol data extends beyond buffer", {
-              field: field["FieldName"],
-              offset: symbolsOffset,
-              length: symbolsLength,
-              bufferSize: symbolBuffer.length,
-              file: this._path,
-              stage: "parseSymbolTable"
-            });
-          }
+          validateFieldMetadata(field, symbolBuffer.length, this._path);
         }
         this._symbolTable = fields.map((field) => {
           const symbolsOffset = parseInt(field["Offset"], 10);
           const symbolsLength = parseInt(field["Length"], 10);
+          const fieldName = field["FieldName"];
+          const neededSymbols = symbolsToKeep ? symbolsToKeep.get(fieldName) : null;
+          const filteringEnabled = neededSymbols !== null;
           const symbols = [];
+          let symbolIndex = 0;
           for (let pointer = symbolsOffset; pointer < symbolsOffset + symbolsLength; pointer++) {
             const typeByte = symbolBuffer[pointer++];
-            switch (typeByte) {
-              case 1: {
-                if (pointer + 4 > symbolBuffer.length) {
-                  throw new QvdCorruptedError("Buffer overflow reading integer symbol", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
-                const value = Buffer.from(byteData).readIntLE(0, byteData.length);
-                pointer += 3;
-                symbols.push(QvdSymbol.fromIntValue(value));
-                break;
-              }
-              case 2: {
-                if (pointer + 8 > symbolBuffer.length) {
-                  throw new QvdCorruptedError("Buffer overflow reading double symbol", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
-                const value = Buffer.from(byteData).readDoubleLE(0);
-                pointer += 7;
-                symbols.push(QvdSymbol.fromDoubleValue(value));
-                break;
-              }
-              case 4: {
-                const byteData = [];
-                const maxStringLength = 1048576;
-                while (pointer < symbolBuffer.length && symbolBuffer[pointer] !== 0) {
-                  if (byteData.length >= maxStringLength) {
-                    throw new QvdCorruptedError("String symbol exceeds maximum length", {
-                      field: field["FieldName"],
-                      maxLength: maxStringLength,
-                      file: this._path,
-                      stage: "parseSymbolTable"
-                    });
-                  }
-                  byteData.push(symbolBuffer[pointer++]);
-                }
-                if (pointer >= symbolBuffer.length) {
-                  throw new QvdCorruptedError("String symbol not null-terminated", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const value = Buffer.from(byteData).toString("utf-8");
-                symbols.push(QvdSymbol.fromStringValue(value));
-                break;
-              }
-              case 5: {
-                if (pointer + 4 > symbolBuffer.length) {
-                  throw new QvdCorruptedError("Buffer overflow reading dual integer symbol", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
-                const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
-                pointer += 4;
-                const stringByteData = [];
-                const maxStringLength = 1048576;
-                while (pointer < symbolBuffer.length && symbolBuffer[pointer] !== 0) {
-                  if (stringByteData.length >= maxStringLength) {
-                    throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
-                      field: field["FieldName"],
-                      maxLength: maxStringLength,
-                      file: this._path,
-                      stage: "parseSymbolTable"
-                    });
-                  }
-                  stringByteData.push(symbolBuffer[pointer++]);
-                }
-                if (pointer >= symbolBuffer.length) {
-                  throw new QvdCorruptedError("Dual string symbol not null-terminated", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const stringValue = Buffer.from(stringByteData).toString("utf-8");
-                symbols.push(QvdSymbol.fromDualIntValue(intValue, stringValue));
-                break;
-              }
-              case 6: {
-                if (pointer + 8 > symbolBuffer.length) {
-                  throw new QvdCorruptedError("Buffer overflow reading dual double symbol", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
-                const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
-                pointer += 8;
-                const stringByteData = [];
-                const maxStringLength = 1048576;
-                while (pointer < symbolBuffer.length && symbolBuffer[pointer] !== 0) {
-                  if (stringByteData.length >= maxStringLength) {
-                    throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
-                      field: field["FieldName"],
-                      maxLength: maxStringLength,
-                      file: this._path,
-                      stage: "parseSymbolTable"
-                    });
-                  }
-                  stringByteData.push(symbolBuffer[pointer++]);
-                }
-                if (pointer >= symbolBuffer.length) {
-                  throw new QvdCorruptedError("Dual string symbol not null-terminated", {
-                    field: field["FieldName"],
-                    pointer,
-                    bufferSize: symbolBuffer.length,
-                    file: this._path,
-                    stage: "parseSymbolTable"
-                  });
-                }
-                const stringValue = Buffer.from(stringByteData).toString("utf-8");
-                symbols.push(QvdSymbol.fromDualDoubleValue(doubleValue, stringValue));
-                break;
-              }
-              default: {
-                throw new QvdParseError("Unknown symbol type byte", {
-                  typeByte: typeByte.toString(16),
-                  offset: pointer,
-                  file: this._path,
-                  stage: "parseSymbolTable"
-                });
-              }
-            }
+            const shouldKeepSymbol = !filteringEnabled || !!(neededSymbols && neededSymbols.has(symbolIndex));
+            const { symbol, bytesRead } = parseSymbol(
+              typeByte,
+              symbolBuffer,
+              pointer,
+              symbolBuffer.length,
+              fieldName,
+              this._path,
+              shouldKeepSymbol
+            );
+            symbols.push(symbol);
+            pointer += bytesRead - 1;
+            symbolIndex++;
           }
           return symbols;
         });
@@ -1006,10 +1447,7 @@ var init_QvdFileReader = __esm({
        * @return {Number} The integer value
        */
       _convertBitsToInt32(bits) {
-        if (bits.length === 0) {
-          return 0;
-        }
-        return bits.reduce((value, bit, index) => value += bit * Math.pow(2, index), 0);
+        return convertBitsToInt32(bits);
       }
       /**
        * Parses the bit stuffed index table of the QVD file. This method is part of the parsing process
@@ -1032,105 +1470,21 @@ var init_QvdFileReader = __esm({
           fields = [fields];
         }
         const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
-        if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
-          throw new QvdCorruptedError("Invalid record byte size", {
-            recordSize,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
         const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
-          throw new QvdCorruptedError("Invalid number of records", {
-            totalRows,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
-        if (recordSize === 0 && totalRows > 0) {
-          throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
-            recordSize,
-            totalRows,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
-        if (recordSize > 1048576) {
-          throw new QvdCorruptedError("Record byte size exceeds maximum", {
-            recordSize,
-            maxSize: 1048576,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
         const rowsToLoad = maxRows !== null ? Math.min(maxRows, totalRows) : totalRows;
         const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
-        if (isNaN(indexTableLength) || !Number.isSafeInteger(indexTableLength) || indexTableLength < 0) {
-          throw new QvdCorruptedError("Invalid index table length", {
-            length: indexTableLength,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
-        if (this._indexTableOffset > this._buffer.length) {
-          throw new QvdCorruptedError("Index table offset beyond buffer", {
-            indexTableOffset: this._indexTableOffset,
-            bufferSize: this._buffer.length,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
-        const maxReasonableOverage = 100 * 1024 * 1024;
-        if (indexTableLength > this._buffer.length + maxReasonableOverage) {
-          throw new QvdCorruptedError("Index table length unreasonably large", {
-            indexTableLength,
-            bufferSize: this._buffer.length,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
-        const requiredIndexBytes = rowsToLoad * recordSize;
-        if (indexTableLength < requiredIndexBytes) {
-          throw new QvdCorruptedError("Index table length smaller than required", {
-            indexTableLength,
-            requiredBytes: requiredIndexBytes,
-            rowsToLoad,
-            recordSize,
-            file: this._path,
-            stage: "parseIndexTable"
-          });
-        }
+        validateIndexTableMetadata(
+          recordSize,
+          totalRows,
+          indexTableLength,
+          this._indexTableOffset,
+          this._buffer.length,
+          rowsToLoad,
+          this._path
+        );
         const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
         for (const field of fields) {
-          const bitOffset = parseInt(field["BitOffset"], 10);
-          const bitWidth = parseInt(field["BitWidth"], 10);
-          if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
-            throw new QvdCorruptedError("Invalid bit offset", {
-              field: field["FieldName"],
-              bitOffset,
-              file: this._path,
-              stage: "parseIndexTable"
-            });
-          }
-          if (isNaN(bitWidth) || !Number.isSafeInteger(bitWidth) || bitWidth < 0) {
-            throw new QvdCorruptedError("Invalid bit width", {
-              field: field["FieldName"],
-              bitWidth,
-              file: this._path,
-              stage: "parseIndexTable"
-            });
-          }
-          const recordSizeInBits = recordSize * 8;
-          if (bitOffset + bitWidth > recordSizeInBits) {
-            throw new QvdCorruptedError("Bit field extends beyond record size", {
-              field: field["FieldName"],
-              bitOffset,
-              bitWidth,
-              recordSizeInBits,
-              file: this._path,
-              stage: "parseIndexTable"
-            });
-          }
+          validateFieldBitMetadata(field, recordSize, this._path);
         }
         this._indexTable = [];
         for (let pointer = 0, rowCount = 0; pointer < indexBuffer.length && rowCount < rowsToLoad; pointer += recordSize, rowCount++) {
@@ -1172,7 +1526,19 @@ var init_QvdFileReader = __esm({
       async load(maxRows = null) {
         await this._readData(maxRows);
         await this._parseHeader();
-        await this._parseSymbolTable();
+        let symbolsToKeep = null;
+        if (maxRows !== null && this._header) {
+          const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
+          const SYMBOL_FILTERING_THRESHOLD = 50 * 1024 * 1024;
+          if (symbolTableLength > SYMBOL_FILTERING_THRESHOLD) {
+            symbolsToKeep = await this._analyzeIndexTableSymbolUsage(maxRows);
+            const totalSymbols = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
+            console.log(
+              `[Phase 2.5 Optimization] Using stream-and-skip parsing: keeping ${totalSymbols} symbols from ${(symbolTableLength / 1024 / 1024).toFixed(1)}MB symbol table`
+            );
+          }
+        }
+        await this._parseSymbolTable(symbolsToKeep, maxRows);
         await this._parseIndexTable(maxRows);
         assert(this._header, "The QVD file header has not been parsed.");
         assert(this._symbolTable, "The QVD file symbol table has not been parsed.");
@@ -1620,11 +1986,17 @@ var init_QvdDataFrame = __esm({
        * @param {Object} [options] Optional loading options.
        * @param {number|null} [options.maxRows] The maximum number of rows to load. If not specified, all rows are loaded.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
+       *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
+       *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
       static async fromQvd(path3, options = {}) {
         const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
-        const readerOptions = { allowedDir: options.allowedDir };
+        const readerOptions = {
+          allowedDir: options.allowedDir,
+          memorySafetyFactor: options.memorySafetyFactor
+        };
         return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
       }
       /**