qvdjs 2.0.1 → 2.0.2

package/dist/index.cjs

@@ -5,7 +5,7 @@ var path2 = require('path');
 var assert3 = require('assert');
 var crypto2 = require('crypto');
 var promises = require('timers/promises');
-var xml2 = require('xml2js');
+var xml = require('xml2js');
 var os = require('os');
 var v8 = require('v8');
 var util = require('util');
@@ -16,7 +16,7 @@ var fs2__default = /*#__PURE__*/_interopDefault(fs2);
 var path2__default = /*#__PURE__*/_interopDefault(path2);
 var assert3__default = /*#__PURE__*/_interopDefault(assert3);
 var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
-var xml2__default = /*#__PURE__*/_interopDefault(xml2);
+var xml__default = /*#__PURE__*/_interopDefault(xml);
 var os__default = /*#__PURE__*/_interopDefault(os);
 var v8__default = /*#__PURE__*/_interopDefault(v8);
 
@@ -1802,7 +1802,7 @@ var init_QvdFileWriter = __esm({
             });
           }
         }
-        const builder = new xml2__default.default.Builder({
+        const builder = new xml__default.default.Builder({
           renderOpts: {
             pretty: true,
             newline: "\r\n",
@@ -2364,6 +2364,13 @@ var init_memoryUtils = __esm({
 });
 
 // src/util/validationUtils.js
+function headerInteger(value) {
+  if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
+    return NaN;
+  }
+  const number = Number(value);
+  return number === 0 ? 0 : number;
+}
 function validateHeaderStructure(headerObj, filePath, stage) {
   const tableHeader = headerObj?.["QvdTableHeader"];
   if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
@@ -2373,7 +2380,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
       stage
     });
   }
-  const symbolTableLength = parseInt(tableHeader["Offset"], 10);
+  const symbolTableLength = headerInteger(tableHeader["Offset"]);
   if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
     throw new exports.QvdCorruptedError("Invalid symbol table offset", {
       offset: tableHeader["Offset"],
@@ -2381,6 +2388,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
       stage
     });
   }
+  const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
+  const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
+  if (fieldList.length === 0) {
+    throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
+      file: filePath,
+      stage
+    });
+  }
+  const malformedIndex = fieldList.findIndex(
+    (field) => field === null || typeof field !== "object" || Array.isArray(field)
+  );
+  if (malformedIndex !== -1) {
+    throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
+      fieldIndex: malformedIndex,
+      fieldCount: fieldList.length,
+      file: filePath,
+      stage
+    });
+  }
+  const seen = /* @__PURE__ */ new Map();
+  fieldList.forEach((field, fieldIndex) => {
+    const name = field["FieldName"];
+    if (typeof name !== "string" || name === "") {
+      throw new exports.QvdCorruptedError("The QVD file header declares a field with no usable name", {
+        fieldIndex,
+        fieldName: name,
+        file: filePath,
+        stage
+      });
+    }
+    if (seen.has(name)) {
+      throw new exports.QvdCorruptedError("The QVD file header declares two fields with the same name", {
+        field: name,
+        fieldIndexes: [seen.get(name), fieldIndex],
+        file: filePath,
+        stage
+      });
+    }
+    seen.set(name, fieldIndex);
+  });
+  return fieldList;
 }
 function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
   const heapLimit = getHeapLimit();
@@ -2426,8 +2474,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
   }
 }
 function validateFieldMetadata(field, symbolBufferLength, filePath) {
-  const symbolsOffset = parseInt(field["Offset"], 10);
-  const symbolsLength = parseInt(field["Length"], 10);
+  const symbolsOffset = headerInteger(field["Offset"]);
+  const symbolsLength = headerInteger(field["Length"]);
   if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
     throw new exports.QvdCorruptedError("Invalid symbol offset", {
       field: field["FieldName"],
@@ -2455,6 +2503,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
     });
   }
 }
+function firstOverlap(ranges) {
+  const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
+  let furthest = null;
+  for (const range of claimed) {
+    if (furthest !== null && range.start < furthest.start + furthest.length) {
+      return [furthest, range];
+    }
+    if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
+      furthest = range;
+    }
+  }
+  return null;
+}
+function validateSymbolAreas(fields, filePath) {
+  const overlap = firstOverlap(
+    fields.map((field) => ({
+      field: field["FieldName"],
+      start: headerInteger(field["Offset"]),
+      length: headerInteger(field["Length"])
+    }))
+  );
+  if (overlap !== null) {
+    const [first, second] = overlap;
+    throw new exports.QvdCorruptedError("Symbol areas overlap", {
+      field: second.field,
+      offset: second.start,
+      length: second.length,
+      overlaps: first.field,
+      overlapsOffset: first.start,
+      overlapsLength: first.length,
+      file: filePath,
+      stage: "parseSymbolTable"
+    });
+  }
+}
+function validateBitFields(fields, filePath) {
+  const overlap = firstOverlap(
+    fields.map((field) => ({
+      field: field["FieldName"],
+      start: headerInteger(field["BitOffset"]),
+      length: headerInteger(field["BitWidth"])
+    }))
+  );
+  if (overlap !== null) {
+    const [first, second] = overlap;
+    throw new exports.QvdCorruptedError("Bit fields overlap", {
+      field: second.field,
+      bitOffset: second.start,
+      bitWidth: second.length,
+      overlaps: first.field,
+      overlapsBitOffset: first.start,
+      overlapsBitWidth: first.length,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+}
 function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
   if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
     throw new exports.QvdCorruptedError("Invalid number of records", {
@@ -2464,14 +2569,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
     });
   }
 }
-function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
+function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
   if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
     throw new exports.QvdCorruptedError("Invalid record byte size", {
       recordSize,
       file: filePath,
-      stage: "parseIndexTable"
+      stage
     });
   }
+}
+function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
+  validateRecordSize(recordSize, filePath);
   validateRecordCount(totalRows, filePath);
   if (recordSize === 0 && totalRows > 0) {
     throw new exports.QvdCorruptedError("Record byte size cannot be zero when records exist", {
@@ -2555,8 +2663,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
   }
 }
 function validateFieldBitMetadata(field, recordSize, filePath) {
-  const bitOffset = parseInt(field["BitOffset"], 10);
-  const bitWidth = parseInt(field["BitWidth"], 10);
+  const bitOffset = headerInteger(field["BitOffset"]);
+  const bitWidth = headerInteger(field["BitWidth"]);
   if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
     throw new exports.QvdCorruptedError("Invalid bit offset", {
       field: field["FieldName"],
@@ -2573,7 +2681,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
-  const bias = parseInt(field["Bias"], 10);
+  const bias = headerInteger(field["Bias"]);
   if (isNaN(bias) || !Number.isSafeInteger(bias)) {
     throw new exports.QvdCorruptedError("Invalid bias", {
       field: field["FieldName"],
@@ -2600,6 +2708,14 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
+  if (bias !== 0 && bias !== -2) {
+    throw new exports.QvdCorruptedError("Bias is neither 0 nor -2", {
+      field: field["FieldName"],
+      bias,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   const recordSizeInBits = recordSize * 8;
   if (bitOffset + bitWidth > recordSizeInBits) {
     throw new exports.QvdCorruptedError("Bit field extends beyond record size", {
@@ -2617,11 +2733,16 @@ var init_validationUtils = __esm({
     init_QvdErrors();
     init_memoryUtils();
     init_bitUtils();
+    __name(headerInteger, "headerInteger");
     __name(validateHeaderStructure, "validateHeaderStructure");
     __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
     __name(validateSymbolTableSize, "validateSymbolTableSize");
     __name(validateFieldMetadata, "validateFieldMetadata");
+    __name(firstOverlap, "firstOverlap");
+    __name(validateSymbolAreas, "validateSymbolAreas");
+    __name(validateBitFields, "validateBitFields");
     __name(validateRecordCount, "validateRecordCount");
+    __name(validateRecordSize, "validateRecordSize");
     __name(validateIndexTableMetadata, "validateIndexTableMetadata");
     __name(validateFieldBitMetadata, "validateFieldBitMetadata");
   }
@@ -3253,6 +3374,31 @@ async function* chunksFrom(handle, chunkSize, failed) {
     position += bytesRead;
   }
 }
+async function parseHeaderXml(text, file, stage) {
+  let parsed;
+  try {
+    parsed = await xml__default.default.parseStringPromise(text, { explicitArray: false });
+  } catch (error) {
+    throw new exports.QvdParseError(
+      "The XML header could not be parsed.",
+      {
+        // The first line of the parser's message. The rest is the line and column it stopped at, which
+        // stay on `cause`.
+        reason: String(
+          /** @type {any} */
+          error?.message ?? error
+        ).split("\n")[0],
+        file,
+        stage
+      },
+      { cause: error }
+    );
+  }
+  if (!parsed) {
+    throw new exports.QvdParseError("The XML header could not be parsed.", { file, stage });
+  }
+  return parsed;
+}
 var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
@@ -3273,6 +3419,7 @@ var init_QvdFileReader = __esm({
     ANALYSIS_SLICE_ROWS = 65536;
     COUNT_SYMBOLS_PAST = 65536;
     __name(chunksFrom, "chunksFrom");
+    __name(parseHeaderXml, "parseHeaderXml");
     exports.QvdFileReader = class {
       static {
         __name(this, "QvdFileReader");
@@ -3508,29 +3655,19 @@ var init_QvdFileReader = __esm({
         const headerBuffer = Buffer.concat(headerChunks);
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
-        const headerObj = await xml2__default.default.parseStringPromise(headerXml, { explicitArray: false });
-        if (!headerObj) {
-          throw new exports.QvdParseError("The XML header could not be parsed.", {
-            file: this._path,
-            stage: "readData"
-          });
-        }
-        validateHeaderStructure(headerObj, this._path, "readData");
+        const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
+        const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
         const symbolTableOffset = headerEndIndex;
-        const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
+        const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
         const indexTableOffset = symbolTableOffset + symbolTableLength;
-        const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
-        const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
+        const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
+        const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
         if (headerOnly) {
           this._buffer = headerBuffer.subarray(0, headerEndIndex);
           this._emitProgress("read", 1, 1);
           return;
         }
-        let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
-        if (headerFields && !Array.isArray(headerFields)) {
-          headerFields = [headerFields];
-        }
-        const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
+        const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
         const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
           (value) => Number.isSafeInteger(value) && value >= 0
         );
@@ -3562,20 +3699,8 @@ var init_QvdFileReader = __esm({
         }
         const rowsToLoad = windowRows;
         validateSymbolTableSizeEarly(symbolTableLength, this._path);
-        for (const [name, value] of [
-          ["Offset", symbolTableLength],
-          ["RecordByteSize", recordSize],
-          ["NoOfRecords", totalRows]
-        ]) {
-          if (!Number.isSafeInteger(value) || Number(value) < 0) {
-            throw new exports.QvdCorruptedError(`Invalid header value: ${name}`, {
-              name,
-              value,
-              file: this._path,
-              stage: "readData"
-            });
-          }
-        }
+        validateRecordSize(recordSize, this._path, "readData");
+        validateRecordCount(totalRows, this._path, "readData");
         const skippedIndexBytes = resolved.offset * recordSize;
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
@@ -3674,36 +3799,11 @@ var init_QvdFileReader = __esm({
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
         this._fieldBitMetadataValidated = false;
-        this._header = await xml2__default.default.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
-        if (!this._header) {
-          throw new exports.QvdParseError("The XML header could not be parsed.", {
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
-        validateHeaderStructure(this._header, this._path, "parseHeader");
-        const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
-        const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
-        if (fieldList.length === 0) {
-          throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
-        const malformedIndex = fieldList.findIndex(
-          (field) => field === null || typeof field !== "object" || Array.isArray(field)
-        );
-        if (malformedIndex !== -1) {
-          throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
-            fieldIndex: malformedIndex,
-            fieldCount: fieldList.length,
-            file: this._path,
-            stage: "parseHeader"
-          });
-        }
+        this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
+        const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
         this._headerOffset = headerBeginIndex;
         this._symbolTableOffset = headerEndIndex;
-        this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
+        this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
         this._allFields = fieldList;
         this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
       }
@@ -3735,9 +3835,9 @@ var init_QvdFileReader = __esm({
         }
         const allFields = this._allFields;
         const fields = this._selectedFields;
-        const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
+        const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
+        const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
         const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
         validateIndexTableMetadata(
           recordSize,
@@ -3760,6 +3860,7 @@ var init_QvdFileReader = __esm({
           for (const field of allFields) {
             validateFieldBitMetadata(field, recordSize, this._path);
           }
+          validateBitFields(allFields, this._path);
           this._fieldBitMetadataValidated = true;
         }
         assert3__default.default(
@@ -3790,10 +3891,10 @@ var init_QvdFileReader = __esm({
           this._throwIfAborted();
           const needed = /* @__PURE__ */ new Set();
           symbolUsage[position] = needed;
-          const bitOffset = parseInt(field["BitOffset"], 10);
-          const bitWidth = parseInt(field["BitWidth"], 10);
-          const bias = parseInt(field["Bias"], 10);
-          const length = parseInt(field["Length"], 10);
+          const bitOffset = headerInteger(field["BitOffset"]);
+          const bitWidth = headerInteger(field["BitWidth"]);
+          const bias = headerInteger(field["Bias"]);
+          const length = headerInteger(field["Length"]);
           let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
           let counted = false;
           for (let first = 0; first < rowsToLoad; first += sliceRows) {
@@ -3846,11 +3947,11 @@ var init_QvdFileReader = __esm({
         );
         const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
         validateFieldMetadata(field, symbolBuffer.length, this._path);
-        const offset = parseInt(field["Offset"], 10);
+        const offset = headerInteger(field["Offset"]);
         return countFieldSymbols(
           symbolBuffer,
           offset,
-          offset + parseInt(field["Length"], 10),
+          offset + headerInteger(field["Length"]),
           field["FieldName"],
           this._path
         );
@@ -3885,7 +3986,7 @@ var init_QvdFileReader = __esm({
         const fields = this._selectedFields;
         const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
         const symbolTableSize = symbolBuffer.length;
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
         validateSymbolTableSize(symbolTableSize, this._path, totalRows);
         if (this._headerMatchesFile) {
           validateMemoryAvailability(
@@ -3903,10 +4004,11 @@ var init_QvdFileReader = __esm({
         for (const field of allFields) {
           validateFieldMetadata(field, symbolBuffer.length, this._path);
         }
+        validateSymbolAreas(allFields, this._path);
         this._symbolTable = fields.map((field, position) => {
           this._throwIfAborted();
-          const symbolsOffset = parseInt(field["Offset"], 10);
-          const symbolsLength = parseInt(field["Length"], 10);
+          const symbolsOffset = headerInteger(field["Offset"]);
+          const symbolsLength = headerInteger(field["Length"]);
           const symbols = parseFieldSymbols(
             symbolBuffer,
             symbolsOffset,
@@ -3958,9 +4060,9 @@ var init_QvdFileReader = __esm({
             indexBuffer,
             recordSize,
             rowsToLoad,
-            parseInt(field["BitOffset"], 10),
-            parseInt(field["BitWidth"], 10),
-            parseInt(field["Bias"], 10),
+            headerInteger(field["BitOffset"]),
+            headerInteger(field["BitWidth"]),
+            headerInteger(field["Bias"]),
             new Int32Array(rowsToLoad),
             { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
           );
@@ -3990,23 +4092,13 @@ var init_QvdFileReader = __esm({
         await this._parseHeader();
         this._emitProgress("header", 1, 1);
         this._throwIfAborted();
-        assert3__default.default(this._header, "The QVD file header has not been parsed.");
+        assert3__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
         const header = this._header["QvdTableHeader"];
-        let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
-        if (!Array.isArray(fields)) {
-          fields = [fields];
-        }
-        if (fields.length === 0) {
-          throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
-            file: this._path,
-            stage: "readMetadata"
-          });
-        }
-        const columns = fields.map((field) => field["FieldName"]);
-        const rowCount = parseInt(header["NoOfRecords"], 10);
+        const columns = this._allFields.map((field) => field["FieldName"]);
+        const rowCount = headerInteger(header["NoOfRecords"]);
         validateRecordCount(rowCount, this._path, "readMetadata");
         const shape = new exports.QvdDataFrame([], columns, header, {
-          symbolTableBytes: parseInt(header["Offset"], 10),
+          symbolTableBytes: headerInteger(header["Offset"]),
           totalRows: rowCount,
           rowsLoaded: 0,
           symbolFiltering: false,
@@ -4171,8 +4263,8 @@ var init_QvdFileReader = __esm({
         this._emitProgress("header", 1, 1);
         this._throwIfAborted();
         assert3__default.default(this._header, "The QVD file header has not been parsed.");
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
+        const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
+        const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
         const resolved = resolveWindow(window, totalRows);
         const rowsAvailable = resolved.limit;
         let symbolsToKeep = null;