qvdjs 2.0.1 → 2.0.2

package/dist/index.js

@@ -3,7 +3,7 @@ import path2 from 'path';
 import assert3 from 'assert';
 import crypto2 from 'crypto';
 import { setTimeout } from 'timers/promises';
-import xml2 from 'xml2js';
+import xml from 'xml2js';
 import os from 'os';
 import v8 from 'v8';
 import { types } from 'util';
@@ -1790,7 +1790,7 @@ var init_QvdFileWriter = __esm({
             });
           }
         }
-        const builder = new xml2.Builder({
+        const builder = new xml.Builder({
           renderOpts: {
             pretty: true,
             newline: "\r\n",
@@ -2352,6 +2352,13 @@ var init_memoryUtils = __esm({
 });
 
 // src/util/validationUtils.js
+function headerInteger(value) {
+  if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
+    return NaN;
+  }
+  const number = Number(value);
+  return number === 0 ? 0 : number;
+}
 function validateHeaderStructure(headerObj, filePath, stage) {
   const tableHeader = headerObj?.["QvdTableHeader"];
   if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
@@ -2361,7 +2368,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
       stage
     });
   }
-  const symbolTableLength = parseInt(tableHeader["Offset"], 10);
+  const symbolTableLength = headerInteger(tableHeader["Offset"]);
   if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
     throw new QvdCorruptedError("Invalid symbol table offset", {
       offset: tableHeader["Offset"],
@@ -2369,6 +2376,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
       stage
     });
   }
+  const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
+  const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
+  if (fieldList.length === 0) {
+    throw new QvdCorruptedError("The QVD file header declares no fields", {
+      file: filePath,
+      stage
+    });
+  }
+  const malformedIndex = fieldList.findIndex(
+    (field) => field === null || typeof field !== "object" || Array.isArray(field)
+  );
+  if (malformedIndex !== -1) {
+    throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
+      fieldIndex: malformedIndex,
+      fieldCount: fieldList.length,
+      file: filePath,
+      stage
+    });
+  }
+  const seen = /* @__PURE__ */ new Map();
+  fieldList.forEach((field, fieldIndex) => {
+    const name = field["FieldName"];
+    if (typeof name !== "string" || name === "") {
+      throw new QvdCorruptedError("The QVD file header declares a field with no usable name", {
+        fieldIndex,
+        fieldName: name,
+        file: filePath,
+        stage
+      });
+    }
+    if (seen.has(name)) {
+      throw new QvdCorruptedError("The QVD file header declares two fields with the same name", {
+        field: name,
+        fieldIndexes: [seen.get(name), fieldIndex],
+        file: filePath,
+        stage
+      });
+    }
+    seen.set(name, fieldIndex);
+  });
+  return fieldList;
 }
 function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
   const heapLimit = getHeapLimit();
@@ -2414,8 +2462,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
   }
 }
 function validateFieldMetadata(field, symbolBufferLength, filePath) {
-  const symbolsOffset = parseInt(field["Offset"], 10);
-  const symbolsLength = parseInt(field["Length"], 10);
+  const symbolsOffset = headerInteger(field["Offset"]);
+  const symbolsLength = headerInteger(field["Length"]);
   if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
     throw new QvdCorruptedError("Invalid symbol offset", {
       field: field["FieldName"],
@@ -2443,6 +2491,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
     });
   }
 }
+function firstOverlap(ranges) {
+  const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
+  let furthest = null;
+  for (const range of claimed) {
+    if (furthest !== null && range.start < furthest.start + furthest.length) {
+      return [furthest, range];
+    }
+    if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
+      furthest = range;
+    }
+  }
+  return null;
+}
+function validateSymbolAreas(fields, filePath) {
+  const overlap = firstOverlap(
+    fields.map((field) => ({
+      field: field["FieldName"],
+      start: headerInteger(field["Offset"]),
+      length: headerInteger(field["Length"])
+    }))
+  );
+  if (overlap !== null) {
+    const [first, second] = overlap;
+    throw new QvdCorruptedError("Symbol areas overlap", {
+      field: second.field,
+      offset: second.start,
+      length: second.length,
+      overlaps: first.field,
+      overlapsOffset: first.start,
+      overlapsLength: first.length,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+}
+function validateBitFields(fields, filePath) {
+  const overlap = firstOverlap(
+    fields.map((field) => ({
+      field: field["FieldName"],
+      start: headerInteger(field["BitOffset"]),
+      length: headerInteger(field["BitWidth"])
+    }))
+  );
+  if (overlap !== null) {
+    const [first, second] = overlap;
+    throw new QvdCorruptedError("Bit fields overlap", {
+      field: second.field,
+      bitOffset: second.start,
+      bitWidth: second.length,
+      overlaps: first.field,
+      overlapsBitOffset: first.start,
+      overlapsBitWidth: first.length,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+}
 function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
   if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
     throw new QvdCorruptedError("Invalid number of records", {
@@ -2452,14 +2557,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
     });
   }
 }
-function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
+function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
   if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
     throw new QvdCorruptedError("Invalid record byte size", {
       recordSize,
       file: filePath,
-      stage: "parseIndexTable"
+      stage
     });
   }
+}
+function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
+  validateRecordSize(recordSize, filePath);
   validateRecordCount(totalRows, filePath);
   if (recordSize === 0 && totalRows > 0) {
     throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
@@ -2543,8 +2651,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
   }
 }
 function validateFieldBitMetadata(field, recordSize, filePath) {
-  const bitOffset = parseInt(field["BitOffset"], 10);
-  const bitWidth = parseInt(field["BitWidth"], 10);
+  const bitOffset = headerInteger(field["BitOffset"]);
+  const bitWidth = headerInteger(field["BitWidth"]);
   if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
     throw new QvdCorruptedError("Invalid bit offset", {
       field: field["FieldName"],
@@ -2561,7 +2669,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
-  const bias = parseInt(field["Bias"], 10);
+  const bias = headerInteger(field["Bias"]);
   if (isNaN(bias) || !Number.isSafeInteger(bias)) {
     throw new QvdCorruptedError("Invalid bias", {
       field: field["FieldName"],
@@ -2588,6 +2696,14 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
+  if (bias !== 0 && bias !== -2) {
+    throw new QvdCorruptedError("Bias is neither 0 nor -2", {
+      field: field["FieldName"],
+      bias,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   const recordSizeInBits = recordSize * 8;
   if (bitOffset + bitWidth > recordSizeInBits) {
     throw new QvdCorruptedError("Bit field extends beyond record size", {
@@ -2605,11 +2721,16 @@ var init_validationUtils = __esm({
     init_QvdErrors();
     init_memoryUtils();
     init_bitUtils();
+    __name(headerInteger, "headerInteger");
     __name(validateHeaderStructure, "validateHeaderStructure");
     __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
     __name(validateSymbolTableSize, "validateSymbolTableSize");
     __name(validateFieldMetadata, "validateFieldMetadata");
+    __name(firstOverlap, "firstOverlap");
+    __name(validateSymbolAreas, "validateSymbolAreas");
+    __name(validateBitFields, "validateBitFields");
     __name(validateRecordCount, "validateRecordCount");
+    __name(validateRecordSize, "validateRecordSize");
     __name(validateIndexTableMetadata, "validateIndexTableMetadata");
     __name(validateFieldBitMetadata, "validateFieldBitMetadata");
   }
@@ -3241,6 +3362,31 @@ async function* chunksFrom(handle, chunkSize, failed) {
     position += bytesRead;
   }
 }
+async function parseHeaderXml(text, file, stage) {
+  let parsed;
+  try {
+    parsed = await xml.parseStringPromise(text, { explicitArray: false });
+  } catch (error) {
+    throw new QvdParseError(
+      "The XML header could not be parsed.",
+      {
+        // The first line of the parser's message. The rest is the line and column it stopped at, which
+        // stay on `cause`.
+        reason: String(
+          /** @type {any} */
+          error?.message ?? error
+        ).split("\n")[0],
+        file,
+        stage
+      },
+      { cause: error }
+    );
+  }
+  if (!parsed) {
+    throw new QvdParseError("The XML header could not be parsed.", { file, stage });
+  }
+  return parsed;
+}
 var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST, QvdFileReader;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
@@ -3261,6 +3407,7 @@ var init_QvdFileReader = __esm({
     ANALYSIS_SLICE_ROWS = 65536;
     COUNT_SYMBOLS_PAST = 65536;
     __name(chunksFrom, "chunksFrom");
+    __name(parseHeaderXml, "parseHeaderXml");
     QvdFileReader = class {
       static {
         __name(this, "QvdFileReader");
@@ -3496,29 +3643,19 @@ var init_QvdFileReader = __esm({
         const headerBuffer = Buffer.concat(headerChunks);
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
-        const headerObj = await xml2.parseStringPromise(headerXml, { explicitArray: false });
-        if (!headerObj) {
-          throw new QvdParseError("The XML header could not be parsed.", {
-            file: this._path,
-            stage: "readData"
-          });
-        }
-        validateHeaderStructure(headerObj, this._path, "readData");
+        const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
+        const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
         const symbolTableOffset = headerEndIndex;
-        const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
+        const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
         const indexTableOffset = symbolTableOffset + symbolTableLength;
-        const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
-        const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
+        const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
+        const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
         if (headerOnly) {
           this._buffer = headerBuffer.subarray(0, headerEndIndex);
           this._emitProgress("read", 1, 1);
           return;
         }
-        let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
-        if (headerFields && !Array.isArray(headerFields)) {
-          headerFields = [headerFields];
-        }
-        const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
+        const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
         const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
           (value) => Number.isSafeInteger(value) && value >= 0
         );
@@ -3550,20 +3687,8 @@ var init_QvdFileReader = __esm({
         }
         const rowsToLoad = windowRows;
         validateSymbolTableSizeEarly(symbolTableLength, this._path);
-        for (const [name, value] of [
-          ["Offset", symbolTableLength],
-          ["RecordByteSize", recordSize],
-          ["NoOfRecords", totalRows]
-        ]) {
-          if (!Number.isSafeInteger(value) || Number(value) < 0) {
-            throw new QvdCorruptedError(`Invalid header value: ${name}`, {
-              name,
-              value,
-              file: this._path,
-              stage: "readData"
-            });
-          }
-        }
+        validateRecordSize(recordSize, this._path, "readData");
+        validateRecordCount(totalRows, this._path, "readData");
         const skippedIndexBytes = resolved.offset * recordSize;
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
@@ -3662,36 +3787,11 @@ var init_QvdFileReader = __esm({
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
         this._fieldBitMetadataValidated = false;
-        this._header = await xml2.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
-        if (!this._header) {
-          throw new QvdParseError("The XML header could not be parsed.", {
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
-        validateHeaderStructure(this._header, this._path, "parseHeader");
-        const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
-        const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
-        if (fieldList.length === 0) {
-          throw new QvdCorruptedError("The QVD file header declares no fields", {
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
-        const malformedIndex = fieldList.findIndex(
-          (field) => field === null || typeof field !== "object" || Array.isArray(field)
-        );
-        if (malformedIndex !== -1) {
-          throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
-            fieldIndex: malformedIndex,
-            fieldCount: fieldList.length,
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
+        this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
+        const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
         this._headerOffset = headerBeginIndex;
         this._symbolTableOffset = headerEndIndex;
-        this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
+        this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
         this._allFields = fieldList;
         this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
       }
@@ -3723,9 +3823,9 @@ var init_QvdFileReader = __esm({
         }
         const allFields = this._allFields;
         const fields = this._selectedFields;
-        const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
+        const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
+        const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
         const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
         validateIndexTableMetadata(
           recordSize,
@@ -3748,6 +3848,7 @@ var init_QvdFileReader = __esm({
           for (const field of allFields) {
             validateFieldBitMetadata(field, recordSize, this._path);
           }
+          validateBitFields(allFields, this._path);
           this._fieldBitMetadataValidated = true;
         }
         assert3(
@@ -3778,10 +3879,10 @@ var init_QvdFileReader = __esm({
           this._throwIfAborted();
           const needed = /* @__PURE__ */ new Set();
           symbolUsage[position] = needed;
-          const bitOffset = parseInt(field["BitOffset"], 10);
-          const bitWidth = parseInt(field["BitWidth"], 10);
-          const bias = parseInt(field["Bias"], 10);
-          const length = parseInt(field["Length"], 10);
+          const bitOffset = headerInteger(field["BitOffset"]);
+          const bitWidth = headerInteger(field["BitWidth"]);
+          const bias = headerInteger(field["Bias"]);
+          const length = headerInteger(field["Length"]);
           let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
           let counted = false;
           for (let first = 0; first < rowsToLoad; first += sliceRows) {
@@ -3834,11 +3935,11 @@ var init_QvdFileReader = __esm({
         );
         const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
         validateFieldMetadata(field, symbolBuffer.length, this._path);
-        const offset = parseInt(field["Offset"], 10);
+        const offset = headerInteger(field["Offset"]);
         return countFieldSymbols(
           symbolBuffer,
           offset,
-          offset + parseInt(field["Length"], 10),
+          offset + headerInteger(field["Length"]),
           field["FieldName"],
           this._path
         );
@@ -3873,7 +3974,7 @@ var init_QvdFileReader = __esm({
         const fields = this._selectedFields;
         const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
         const symbolTableSize = symbolBuffer.length;
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
         validateSymbolTableSize(symbolTableSize, this._path, totalRows);
         if (this._headerMatchesFile) {
           validateMemoryAvailability(
@@ -3891,10 +3992,11 @@ var init_QvdFileReader = __esm({
         for (const field of allFields) {
           validateFieldMetadata(field, symbolBuffer.length, this._path);
         }
+        validateSymbolAreas(allFields, this._path);
         this._symbolTable = fields.map((field, position) => {
           this._throwIfAborted();
-          const symbolsOffset = parseInt(field["Offset"], 10);
-          const symbolsLength = parseInt(field["Length"], 10);
+          const symbolsOffset = headerInteger(field["Offset"]);
+          const symbolsLength = headerInteger(field["Length"]);
           const symbols = parseFieldSymbols(
             symbolBuffer,
             symbolsOffset,
@@ -3946,9 +4048,9 @@ var init_QvdFileReader = __esm({
             indexBuffer,
             recordSize,
             rowsToLoad,
-            parseInt(field["BitOffset"], 10),
-            parseInt(field["BitWidth"], 10),
-            parseInt(field["Bias"], 10),
+            headerInteger(field["BitOffset"]),
+            headerInteger(field["BitWidth"]),
+            headerInteger(field["Bias"]),
             new Int32Array(rowsToLoad),
             { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
           );
@@ -3978,23 +4080,13 @@ var init_QvdFileReader = __esm({
         await this._parseHeader();
         this._emitProgress("header", 1, 1);
         this._throwIfAborted();
-        assert3(this._header, "The QVD file header has not been parsed.");
+        assert3(this._header && this._allFields, "The QVD file header has not been parsed.");
         const header = this._header["QvdTableHeader"];
-        let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
-        if (!Array.isArray(fields)) {
-          fields = [fields];
-        }
-        if (fields.length === 0) {
-          throw new QvdCorruptedError("The QVD file header declares no fields", {
-            file: this._path,
-            stage: "readMetadata"
-          });
-        }
-        const columns = fields.map((field) => field["FieldName"]);
-        const rowCount = parseInt(header["NoOfRecords"], 10);
+        const columns = this._allFields.map((field) => field["FieldName"]);
+        const rowCount = headerInteger(header["NoOfRecords"]);
         validateRecordCount(rowCount, this._path, "readMetadata");
         const shape = new QvdDataFrame([], columns, header, {
-          symbolTableBytes: parseInt(header["Offset"], 10),
+          symbolTableBytes: headerInteger(header["Offset"]),
           totalRows: rowCount,
           rowsLoaded: 0,
           symbolFiltering: false,
@@ -4159,8 +4251,8 @@ var init_QvdFileReader = __esm({
         this._emitProgress("header", 1, 1);
         this._throwIfAborted();
         assert3(this._header, "The QVD file header has not been parsed.");
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
+        const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
         const resolved = resolveWindow(window, totalRows);
         const rowsAvailable = resolved.limit;
         let symbolsToKeep = null;