qvdjs 2.0.0 → 2.0.1

package/dist/index.js

@@ -1086,8 +1086,12 @@ function fieldGeometry(bitOffset, bitWidth) {
     byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
   };
 }
-function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
+function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
+  const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
   if (bitWidth === 0) {
+    if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
+      refuse2(bounds, 0, bias, bias);
+    }
     out.fill(bias, 0, rowCount);
     return out;
   }
@@ -1101,10 +1105,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
     if (byteCount > 2) acc += buffer[base + 2] * 65536;
     if (byteCount > 3) acc += buffer[base + 3] * 16777216;
     if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
-    out[row] = Math.floor(acc / divisor) % modulus + bias;
+    const index = Math.floor(acc / divisor) % modulus + bias;
+    if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
+      refuse2(bounds, row, index, bias);
+    }
+    out[row] = index;
   }
   return out;
 }
+function refuse2(bounds, row, index, bias) {
+  throw new QvdCorruptedError("Symbol index out of range", {
+    field: bounds.field,
+    row: bounds.firstRow + row,
+    symbolIndex: index,
+    symbolCount: bounds.symbolCount,
+    bias,
+    file: bounds.file,
+    stage: "parseIndexTable"
+  });
+}
 function writeBitField(buffer, recordBase, geometry, value) {
   const { byteStart, shift, byteCount } = geometry;
   if (byteCount === 0) {
@@ -1121,10 +1140,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
 var MAX_BIT_WIDTH, POW2;
 var init_bitUtils = __esm({
   "src/util/bitUtils.js"() {
+    init_QvdErrors();
     MAX_BIT_WIDTH = 31;
     POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
     __name(fieldGeometry, "fieldGeometry");
     __name(decodeIndexColumn, "decodeIndexColumn");
+    __name(refuse2, "refuse");
     __name(writeBitField, "writeBitField");
   }
 });
@@ -2558,6 +2579,15 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
+  if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
+    throw new QvdCorruptedError("Bias out of range", {
+      field: field["FieldName"],
+      bias,
+      bitWidth,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   const recordSizeInBits = recordSize * 8;
   if (bitOffset + bitWidth > recordSizeInBits) {
     throw new QvdCorruptedError("Bit field extends beyond record size", {
@@ -2685,7 +2715,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
   }
   return { numbers, texts };
 }
-var MAX_TEXT_BYTES;
+function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
+  return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
+}
+var MAX_TEXT_BYTES, DECODE_NOTHING;
 var init_symbolParser = __esm({
   "src/util/symbolParser.js"() {
     init_QvdErrors();
@@ -2693,6 +2726,8 @@ var init_symbolParser = __esm({
     __name(textEnd, "textEnd");
     __name(overflow, "overflow");
     __name(parseFieldSymbols, "parseFieldSymbols");
+    DECODE_NOTHING = /* @__PURE__ */ new Set();
+    __name(countFieldSymbols, "countFieldSymbols");
   }
 });
 
@@ -3206,7 +3241,7 @@ async function* chunksFrom(handle, chunkSize, failed) {
     position += bytesRead;
   }
 }
-var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
+var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST, QvdFileReader;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
     init_QvdDataFrame();
@@ -3224,6 +3259,7 @@ var init_QvdFileReader = __esm({
     MAX_HEADER_SIZE = 16 * 1024 * 1024;
     READ_CHUNK_SIZE = 512 * 1024 * 1024;
     ANALYSIS_SLICE_ROWS = 65536;
+    COUNT_SYMBOLS_PAST = 65536;
     __name(chunksFrom, "chunksFrom");
     QvdFileReader = class {
       static {
@@ -3625,6 +3661,7 @@ var init_QvdFileReader = __esm({
         }
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
+        this._fieldBitMetadataValidated = false;
         this._header = await xml2.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
         if (!this._header) {
           throw new QvdParseError("The XML header could not be parsed.", {
@@ -3670,8 +3707,8 @@ var init_QvdFileReader = __esm({
        * @param {QvdRowWindow} window The rows of interest, as file row indices.
        * @param {string} stage Stage name for any error raised here.
        * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
-       *   indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
-       *   record, so the decoder always counts from zero.
+       *   indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
+       *   window's first record, file row `firstRow`, so the decoder always counts from zero.
        * @private
        */
       _planIndexTable(window, stage) {
@@ -3717,7 +3754,7 @@ var init_QvdFileReader = __esm({
           rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
           `The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
         );
-        return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
+        return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
       }
       /**
        * Analyzes the index table to determine which symbols are actually needed.
@@ -3744,6 +3781,9 @@ var init_QvdFileReader = __esm({
           const bitOffset = parseInt(field["BitOffset"], 10);
           const bitWidth = parseInt(field["BitWidth"], 10);
           const bias = parseInt(field["Bias"], 10);
+          const length = parseInt(field["Length"], 10);
+          let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
+          let counted = false;
           for (let first = 0; first < rowsToLoad; first += sliceRows) {
             const count = Math.min(sliceRows, rowsToLoad - first);
             decodeIndexColumn(
@@ -3756,15 +3796,53 @@ var init_QvdFileReader = __esm({
               column
             );
             for (let row = 0; row < count; row++) {
-              if (column[row] >= 0) {
+              if (column[row] >= 0 && column[row] < indexLimit) {
                 needed.add(column[row]);
               }
             }
+            if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
+              counted = true;
+              indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
+              for (const index of needed) {
+                if (index >= indexLimit) {
+                  needed.delete(index);
+                }
+              }
+            }
           }
           this._emitProgress("symbol-analysis", position + 1, fields.length);
         });
         return symbolUsage;
       }
+      /**
+       * How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
+       *
+       * The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
+       * `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
+       * validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
+       * same way wherever it is met.
+       *
+       * @param {any} field The field's header.
+       * @return {number} Its symbols.
+       * @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
+       * @private
+       */
+      _countFieldSymbols(field) {
+        assert3(
+          this._buffer && this._symbolTableOffset && this._indexTableOffset,
+          "The QVD file has not been read before its symbols were counted."
+        );
+        const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
+        validateFieldMetadata(field, symbolBuffer.length, this._path);
+        const offset = parseInt(field["Offset"], 10);
+        return countFieldSymbols(
+          symbolBuffer,
+          offset,
+          offset + parseInt(field["Length"], 10),
+          field["FieldName"],
+          this._path
+        );
+      }
       /**
        * Parses the symbol table of the QVD file. This method is part of the parsing process
        * and should not be called directly.
@@ -3850,12 +3928,19 @@ var init_QvdFileReader = __esm({
        * `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
        * and how many iterations run. Nothing about the decoder changed to support it.
        *
+       * Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
+       * NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
+       * are built, because every read decodes through this method and a columnar read hands its codes
+       * straight to the caller. Rows outside the window are not decoded, so they are not checked.
+       *
        * @param {QvdRowWindow} window The rows to decode.
+       * @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
        */
       async _parseIndexTable(window) {
-        const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
-        this._rowsDecoded = rowsToLoad;
-        this._indexColumns = fields.map((field, position) => {
+        const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
+        assert3(this._symbolTable, "The QVD file symbol table has not been parsed.");
+        const symbolTable = this._symbolTable;
+        const columns = fields.map((field, position) => {
           this._throwIfAborted();
           const column = decodeIndexColumn(
             indexBuffer,
@@ -3864,11 +3949,14 @@ var init_QvdFileReader = __esm({
             parseInt(field["BitOffset"], 10),
             parseInt(field["BitWidth"], 10),
             parseInt(field["Bias"], 10),
-            new Int32Array(rowsToLoad)
+            new Int32Array(rowsToLoad),
+            { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
           );
           this._emitProgress("index-table", position + 1, fields.length);
           return column;
         });
+        this._indexColumns = columns;
+        this._rowsDecoded = rowsToLoad;
       }
       /**
        * Reads the file's schema and header metadata, without touching the symbol or index tables.
@@ -3997,7 +4085,8 @@ var init_QvdFileReader = __esm({
        * `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
        *
        * A window covering no rows yields nothing at all, rather than one empty frame - so
-       * `for await` over an exhausted offset does nothing, which is what a paging loop wants.
+       * `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
+       * is still checked, as every read of a file's rows checks it.
        *
        * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
        *   The rows to cover, in the same spellings `load()` accepts.
@@ -4015,6 +4104,10 @@ var init_QvdFileReader = __esm({
         const liveRows = { rows: chunkSize * 2, perChunk: 2 };
         const rows = normaliseWindow(window, this._path);
         const prepared = await this._prepare(rows, liveRows);
+        if (prepared.rowsAvailable === 0) {
+          this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
+          return;
+        }
         for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
           this._throwIfAborted();
           const count = Math.min(chunkSize, prepared.rowsAvailable - done);