qvdjs 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3,7 +3,7 @@ import path2 from 'path';
3
3
  import assert3 from 'assert';
4
4
  import crypto2 from 'crypto';
5
5
  import { setTimeout } from 'timers/promises';
6
- import xml2 from 'xml2js';
6
+ import xml from 'xml2js';
7
7
  import os from 'os';
8
8
  import v8 from 'v8';
9
9
  import { types } from 'util';
@@ -1086,8 +1086,12 @@ function fieldGeometry(bitOffset, bitWidth) {
1086
1086
  byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
1087
1087
  };
1088
1088
  }
1089
- function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
1089
+ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
1090
+ const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
1090
1091
  if (bitWidth === 0) {
1092
+ if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
1093
+ refuse2(bounds, 0, bias, bias);
1094
+ }
1091
1095
  out.fill(bias, 0, rowCount);
1092
1096
  return out;
1093
1097
  }
@@ -1101,10 +1105,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
1101
1105
  if (byteCount > 2) acc += buffer[base + 2] * 65536;
1102
1106
  if (byteCount > 3) acc += buffer[base + 3] * 16777216;
1103
1107
  if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
1104
- out[row] = Math.floor(acc / divisor) % modulus + bias;
1108
+ const index = Math.floor(acc / divisor) % modulus + bias;
1109
+ if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
1110
+ refuse2(bounds, row, index, bias);
1111
+ }
1112
+ out[row] = index;
1105
1113
  }
1106
1114
  return out;
1107
1115
  }
1116
+ function refuse2(bounds, row, index, bias) {
1117
+ throw new QvdCorruptedError("Symbol index out of range", {
1118
+ field: bounds.field,
1119
+ row: bounds.firstRow + row,
1120
+ symbolIndex: index,
1121
+ symbolCount: bounds.symbolCount,
1122
+ bias,
1123
+ file: bounds.file,
1124
+ stage: "parseIndexTable"
1125
+ });
1126
+ }
1108
1127
  function writeBitField(buffer, recordBase, geometry, value) {
1109
1128
  const { byteStart, shift, byteCount } = geometry;
1110
1129
  if (byteCount === 0) {
@@ -1121,10 +1140,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
1121
1140
  var MAX_BIT_WIDTH, POW2;
1122
1141
  var init_bitUtils = __esm({
1123
1142
  "src/util/bitUtils.js"() {
1143
+ init_QvdErrors();
1124
1144
  MAX_BIT_WIDTH = 31;
1125
1145
  POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
1126
1146
  __name(fieldGeometry, "fieldGeometry");
1127
1147
  __name(decodeIndexColumn, "decodeIndexColumn");
1148
+ __name(refuse2, "refuse");
1128
1149
  __name(writeBitField, "writeBitField");
1129
1150
  }
1130
1151
  });
@@ -1769,7 +1790,7 @@ var init_QvdFileWriter = __esm({
1769
1790
  });
1770
1791
  }
1771
1792
  }
1772
- const builder = new xml2.Builder({
1793
+ const builder = new xml.Builder({
1773
1794
  renderOpts: {
1774
1795
  pretty: true,
1775
1796
  newline: "\r\n",
@@ -2331,6 +2352,13 @@ var init_memoryUtils = __esm({
2331
2352
  });
2332
2353
 
2333
2354
  // src/util/validationUtils.js
2355
+ function headerInteger(value) {
2356
+ if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
2357
+ return NaN;
2358
+ }
2359
+ const number = Number(value);
2360
+ return number === 0 ? 0 : number;
2361
+ }
2334
2362
  function validateHeaderStructure(headerObj, filePath, stage) {
2335
2363
  const tableHeader = headerObj?.["QvdTableHeader"];
2336
2364
  if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
@@ -2340,7 +2368,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
2340
2368
  stage
2341
2369
  });
2342
2370
  }
2343
- const symbolTableLength = parseInt(tableHeader["Offset"], 10);
2371
+ const symbolTableLength = headerInteger(tableHeader["Offset"]);
2344
2372
  if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
2345
2373
  throw new QvdCorruptedError("Invalid symbol table offset", {
2346
2374
  offset: tableHeader["Offset"],
@@ -2348,6 +2376,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
2348
2376
  stage
2349
2377
  });
2350
2378
  }
2379
+ const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
2380
+ const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
2381
+ if (fieldList.length === 0) {
2382
+ throw new QvdCorruptedError("The QVD file header declares no fields", {
2383
+ file: filePath,
2384
+ stage
2385
+ });
2386
+ }
2387
+ const malformedIndex = fieldList.findIndex(
2388
+ (field) => field === null || typeof field !== "object" || Array.isArray(field)
2389
+ );
2390
+ if (malformedIndex !== -1) {
2391
+ throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
2392
+ fieldIndex: malformedIndex,
2393
+ fieldCount: fieldList.length,
2394
+ file: filePath,
2395
+ stage
2396
+ });
2397
+ }
2398
+ const seen = /* @__PURE__ */ new Map();
2399
+ fieldList.forEach((field, fieldIndex) => {
2400
+ const name = field["FieldName"];
2401
+ if (typeof name !== "string" || name === "") {
2402
+ throw new QvdCorruptedError("The QVD file header declares a field with no usable name", {
2403
+ fieldIndex,
2404
+ fieldName: name,
2405
+ file: filePath,
2406
+ stage
2407
+ });
2408
+ }
2409
+ if (seen.has(name)) {
2410
+ throw new QvdCorruptedError("The QVD file header declares two fields with the same name", {
2411
+ field: name,
2412
+ fieldIndexes: [seen.get(name), fieldIndex],
2413
+ file: filePath,
2414
+ stage
2415
+ });
2416
+ }
2417
+ seen.set(name, fieldIndex);
2418
+ });
2419
+ return fieldList;
2351
2420
  }
2352
2421
  function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
2353
2422
  const heapLimit = getHeapLimit();
@@ -2393,8 +2462,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
2393
2462
  }
2394
2463
  }
2395
2464
  function validateFieldMetadata(field, symbolBufferLength, filePath) {
2396
- const symbolsOffset = parseInt(field["Offset"], 10);
2397
- const symbolsLength = parseInt(field["Length"], 10);
2465
+ const symbolsOffset = headerInteger(field["Offset"]);
2466
+ const symbolsLength = headerInteger(field["Length"]);
2398
2467
  if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
2399
2468
  throw new QvdCorruptedError("Invalid symbol offset", {
2400
2469
  field: field["FieldName"],
@@ -2422,6 +2491,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
2422
2491
  });
2423
2492
  }
2424
2493
  }
2494
+ function firstOverlap(ranges) {
2495
+ const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
2496
+ let furthest = null;
2497
+ for (const range of claimed) {
2498
+ if (furthest !== null && range.start < furthest.start + furthest.length) {
2499
+ return [furthest, range];
2500
+ }
2501
+ if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
2502
+ furthest = range;
2503
+ }
2504
+ }
2505
+ return null;
2506
+ }
2507
+ function validateSymbolAreas(fields, filePath) {
2508
+ const overlap = firstOverlap(
2509
+ fields.map((field) => ({
2510
+ field: field["FieldName"],
2511
+ start: headerInteger(field["Offset"]),
2512
+ length: headerInteger(field["Length"])
2513
+ }))
2514
+ );
2515
+ if (overlap !== null) {
2516
+ const [first, second] = overlap;
2517
+ throw new QvdCorruptedError("Symbol areas overlap", {
2518
+ field: second.field,
2519
+ offset: second.start,
2520
+ length: second.length,
2521
+ overlaps: first.field,
2522
+ overlapsOffset: first.start,
2523
+ overlapsLength: first.length,
2524
+ file: filePath,
2525
+ stage: "parseSymbolTable"
2526
+ });
2527
+ }
2528
+ }
2529
+ function validateBitFields(fields, filePath) {
2530
+ const overlap = firstOverlap(
2531
+ fields.map((field) => ({
2532
+ field: field["FieldName"],
2533
+ start: headerInteger(field["BitOffset"]),
2534
+ length: headerInteger(field["BitWidth"])
2535
+ }))
2536
+ );
2537
+ if (overlap !== null) {
2538
+ const [first, second] = overlap;
2539
+ throw new QvdCorruptedError("Bit fields overlap", {
2540
+ field: second.field,
2541
+ bitOffset: second.start,
2542
+ bitWidth: second.length,
2543
+ overlaps: first.field,
2544
+ overlapsBitOffset: first.start,
2545
+ overlapsBitWidth: first.length,
2546
+ file: filePath,
2547
+ stage: "parseIndexTable"
2548
+ });
2549
+ }
2550
+ }
2425
2551
  function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
2426
2552
  if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
2427
2553
  throw new QvdCorruptedError("Invalid number of records", {
@@ -2431,14 +2557,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
2431
2557
  });
2432
2558
  }
2433
2559
  }
2434
- function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
2560
+ function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
2435
2561
  if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
2436
2562
  throw new QvdCorruptedError("Invalid record byte size", {
2437
2563
  recordSize,
2438
2564
  file: filePath,
2439
- stage: "parseIndexTable"
2565
+ stage
2440
2566
  });
2441
2567
  }
2568
+ }
2569
+ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
2570
+ validateRecordSize(recordSize, filePath);
2442
2571
  validateRecordCount(totalRows, filePath);
2443
2572
  if (recordSize === 0 && totalRows > 0) {
2444
2573
  throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
@@ -2522,8 +2651,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
2522
2651
  }
2523
2652
  }
2524
2653
  function validateFieldBitMetadata(field, recordSize, filePath) {
2525
- const bitOffset = parseInt(field["BitOffset"], 10);
2526
- const bitWidth = parseInt(field["BitWidth"], 10);
2654
+ const bitOffset = headerInteger(field["BitOffset"]);
2655
+ const bitWidth = headerInteger(field["BitWidth"]);
2527
2656
  if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
2528
2657
  throw new QvdCorruptedError("Invalid bit offset", {
2529
2658
  field: field["FieldName"],
@@ -2540,7 +2669,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
2540
2669
  stage: "parseIndexTable"
2541
2670
  });
2542
2671
  }
2543
- const bias = parseInt(field["Bias"], 10);
2672
+ const bias = headerInteger(field["Bias"]);
2544
2673
  if (isNaN(bias) || !Number.isSafeInteger(bias)) {
2545
2674
  throw new QvdCorruptedError("Invalid bias", {
2546
2675
  field: field["FieldName"],
@@ -2558,6 +2687,23 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
2558
2687
  stage: "parseIndexTable"
2559
2688
  });
2560
2689
  }
2690
+ if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
2691
+ throw new QvdCorruptedError("Bias out of range", {
2692
+ field: field["FieldName"],
2693
+ bias,
2694
+ bitWidth,
2695
+ file: filePath,
2696
+ stage: "parseIndexTable"
2697
+ });
2698
+ }
2699
+ if (bias !== 0 && bias !== -2) {
2700
+ throw new QvdCorruptedError("Bias is neither 0 nor -2", {
2701
+ field: field["FieldName"],
2702
+ bias,
2703
+ file: filePath,
2704
+ stage: "parseIndexTable"
2705
+ });
2706
+ }
2561
2707
  const recordSizeInBits = recordSize * 8;
2562
2708
  if (bitOffset + bitWidth > recordSizeInBits) {
2563
2709
  throw new QvdCorruptedError("Bit field extends beyond record size", {
@@ -2575,11 +2721,16 @@ var init_validationUtils = __esm({
2575
2721
  init_QvdErrors();
2576
2722
  init_memoryUtils();
2577
2723
  init_bitUtils();
2724
+ __name(headerInteger, "headerInteger");
2578
2725
  __name(validateHeaderStructure, "validateHeaderStructure");
2579
2726
  __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
2580
2727
  __name(validateSymbolTableSize, "validateSymbolTableSize");
2581
2728
  __name(validateFieldMetadata, "validateFieldMetadata");
2729
+ __name(firstOverlap, "firstOverlap");
2730
+ __name(validateSymbolAreas, "validateSymbolAreas");
2731
+ __name(validateBitFields, "validateBitFields");
2582
2732
  __name(validateRecordCount, "validateRecordCount");
2733
+ __name(validateRecordSize, "validateRecordSize");
2583
2734
  __name(validateIndexTableMetadata, "validateIndexTableMetadata");
2584
2735
  __name(validateFieldBitMetadata, "validateFieldBitMetadata");
2585
2736
  }
@@ -2685,7 +2836,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
2685
2836
  }
2686
2837
  return { numbers, texts };
2687
2838
  }
2688
- var MAX_TEXT_BYTES;
2839
+ function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
2840
+ return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
2841
+ }
2842
+ var MAX_TEXT_BYTES, DECODE_NOTHING;
2689
2843
  var init_symbolParser = __esm({
2690
2844
  "src/util/symbolParser.js"() {
2691
2845
  init_QvdErrors();
@@ -2693,6 +2847,8 @@ var init_symbolParser = __esm({
2693
2847
  __name(textEnd, "textEnd");
2694
2848
  __name(overflow, "overflow");
2695
2849
  __name(parseFieldSymbols, "parseFieldSymbols");
2850
+ DECODE_NOTHING = /* @__PURE__ */ new Set();
2851
+ __name(countFieldSymbols, "countFieldSymbols");
2696
2852
  }
2697
2853
  });
2698
2854
 
@@ -3206,7 +3362,32 @@ async function* chunksFrom(handle, chunkSize, failed) {
3206
3362
  position += bytesRead;
3207
3363
  }
3208
3364
  }
3209
- var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
3365
+ async function parseHeaderXml(text, file, stage) {
3366
+ let parsed;
3367
+ try {
3368
+ parsed = await xml.parseStringPromise(text, { explicitArray: false });
3369
+ } catch (error) {
3370
+ throw new QvdParseError(
3371
+ "The XML header could not be parsed.",
3372
+ {
3373
+ // The first line of the parser's message. The rest is the line and column it stopped at, which
3374
+ // stay on `cause`.
3375
+ reason: String(
3376
+ /** @type {any} */
3377
+ error?.message ?? error
3378
+ ).split("\n")[0],
3379
+ file,
3380
+ stage
3381
+ },
3382
+ { cause: error }
3383
+ );
3384
+ }
3385
+ if (!parsed) {
3386
+ throw new QvdParseError("The XML header could not be parsed.", { file, stage });
3387
+ }
3388
+ return parsed;
3389
+ }
3390
+ var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST, QvdFileReader;
3210
3391
  var init_QvdFileReader = __esm({
3211
3392
  "src/QvdFileReader.js"() {
3212
3393
  init_QvdDataFrame();
@@ -3224,7 +3405,9 @@ var init_QvdFileReader = __esm({
3224
3405
  MAX_HEADER_SIZE = 16 * 1024 * 1024;
3225
3406
  READ_CHUNK_SIZE = 512 * 1024 * 1024;
3226
3407
  ANALYSIS_SLICE_ROWS = 65536;
3408
+ COUNT_SYMBOLS_PAST = 65536;
3227
3409
  __name(chunksFrom, "chunksFrom");
3410
+ __name(parseHeaderXml, "parseHeaderXml");
3228
3411
  QvdFileReader = class {
3229
3412
  static {
3230
3413
  __name(this, "QvdFileReader");
@@ -3460,29 +3643,19 @@ var init_QvdFileReader = __esm({
3460
3643
  const headerBuffer = Buffer.concat(headerChunks);
3461
3644
  const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
3462
3645
  const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
3463
- const headerObj = await xml2.parseStringPromise(headerXml, { explicitArray: false });
3464
- if (!headerObj) {
3465
- throw new QvdParseError("The XML header could not be parsed.", {
3466
- file: this._path,
3467
- stage: "readData"
3468
- });
3469
- }
3470
- validateHeaderStructure(headerObj, this._path, "readData");
3646
+ const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
3647
+ const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
3471
3648
  const symbolTableOffset = headerEndIndex;
3472
- const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
3649
+ const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
3473
3650
  const indexTableOffset = symbolTableOffset + symbolTableLength;
3474
- const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
3475
- const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
3651
+ const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
3652
+ const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
3476
3653
  if (headerOnly) {
3477
3654
  this._buffer = headerBuffer.subarray(0, headerEndIndex);
3478
3655
  this._emitProgress("read", 1, 1);
3479
3656
  return;
3480
3657
  }
3481
- let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
3482
- if (headerFields && !Array.isArray(headerFields)) {
3483
- headerFields = [headerFields];
3484
- }
3485
- const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
3658
+ const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
3486
3659
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
3487
3660
  (value) => Number.isSafeInteger(value) && value >= 0
3488
3661
  );
@@ -3514,20 +3687,8 @@ var init_QvdFileReader = __esm({
3514
3687
  }
3515
3688
  const rowsToLoad = windowRows;
3516
3689
  validateSymbolTableSizeEarly(symbolTableLength, this._path);
3517
- for (const [name, value] of [
3518
- ["Offset", symbolTableLength],
3519
- ["RecordByteSize", recordSize],
3520
- ["NoOfRecords", totalRows]
3521
- ]) {
3522
- if (!Number.isSafeInteger(value) || Number(value) < 0) {
3523
- throw new QvdCorruptedError(`Invalid header value: ${name}`, {
3524
- name,
3525
- value,
3526
- file: this._path,
3527
- stage: "readData"
3528
- });
3529
- }
3530
- }
3690
+ validateRecordSize(recordSize, this._path, "readData");
3691
+ validateRecordCount(totalRows, this._path, "readData");
3531
3692
  const skippedIndexBytes = resolved.offset * recordSize;
3532
3693
  const indexTableBytesToRead = rowsToLoad * recordSize;
3533
3694
  const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
@@ -3625,36 +3786,12 @@ var init_QvdFileReader = __esm({
3625
3786
  }
3626
3787
  const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
3627
3788
  const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
3628
- this._header = await xml2.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
3629
- if (!this._header) {
3630
- throw new QvdParseError("The XML header could not be parsed.", {
3631
- file: this._path,
3632
- stage: "parseHeader"
3633
- });
3634
- }
3635
- validateHeaderStructure(this._header, this._path, "parseHeader");
3636
- const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
3637
- const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
3638
- if (fieldList.length === 0) {
3639
- throw new QvdCorruptedError("The QVD file header declares no fields", {
3640
- file: this._path,
3641
- stage: "parseHeader"
3642
- });
3643
- }
3644
- const malformedIndex = fieldList.findIndex(
3645
- (field) => field === null || typeof field !== "object" || Array.isArray(field)
3646
- );
3647
- if (malformedIndex !== -1) {
3648
- throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
3649
- fieldIndex: malformedIndex,
3650
- fieldCount: fieldList.length,
3651
- file: this._path,
3652
- stage: "parseHeader"
3653
- });
3654
- }
3789
+ this._fieldBitMetadataValidated = false;
3790
+ this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
3791
+ const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
3655
3792
  this._headerOffset = headerBeginIndex;
3656
3793
  this._symbolTableOffset = headerEndIndex;
3657
- this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3794
+ this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
3658
3795
  this._allFields = fieldList;
3659
3796
  this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
3660
3797
  }
@@ -3670,8 +3807,8 @@ var init_QvdFileReader = __esm({
3670
3807
  * @param {QvdRowWindow} window The rows of interest, as file row indices.
3671
3808
  * @param {string} stage Stage name for any error raised here.
3672
3809
  * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
3673
- * indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
3674
- * record, so the decoder always counts from zero.
3810
+ * indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
3811
+ * window's first record, file row `firstRow`, so the decoder always counts from zero.
3675
3812
  * @private
3676
3813
  */
3677
3814
  _planIndexTable(window, stage) {
@@ -3686,9 +3823,9 @@ var init_QvdFileReader = __esm({
3686
3823
  }
3687
3824
  const allFields = this._allFields;
3688
3825
  const fields = this._selectedFields;
3689
- const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
3690
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3691
- const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
3826
+ const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
3827
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3828
+ const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
3692
3829
  const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
3693
3830
  validateIndexTableMetadata(
3694
3831
  recordSize,
@@ -3711,13 +3848,14 @@ var init_QvdFileReader = __esm({
3711
3848
  for (const field of allFields) {
3712
3849
  validateFieldBitMetadata(field, recordSize, this._path);
3713
3850
  }
3851
+ validateBitFields(allFields, this._path);
3714
3852
  this._fieldBitMetadataValidated = true;
3715
3853
  }
3716
3854
  assert3(
3717
3855
  rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
3718
3856
  `The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
3719
3857
  );
3720
- return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
3858
+ return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
3721
3859
  }
3722
3860
  /**
3723
3861
  * Analyzes the index table to determine which symbols are actually needed.
@@ -3741,9 +3879,12 @@ var init_QvdFileReader = __esm({
3741
3879
  this._throwIfAborted();
3742
3880
  const needed = /* @__PURE__ */ new Set();
3743
3881
  symbolUsage[position] = needed;
3744
- const bitOffset = parseInt(field["BitOffset"], 10);
3745
- const bitWidth = parseInt(field["BitWidth"], 10);
3746
- const bias = parseInt(field["Bias"], 10);
3882
+ const bitOffset = headerInteger(field["BitOffset"]);
3883
+ const bitWidth = headerInteger(field["BitWidth"]);
3884
+ const bias = headerInteger(field["Bias"]);
3885
+ const length = headerInteger(field["Length"]);
3886
+ let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
3887
+ let counted = false;
3747
3888
  for (let first = 0; first < rowsToLoad; first += sliceRows) {
3748
3889
  const count = Math.min(sliceRows, rowsToLoad - first);
3749
3890
  decodeIndexColumn(
@@ -3756,15 +3897,53 @@ var init_QvdFileReader = __esm({
3756
3897
  column
3757
3898
  );
3758
3899
  for (let row = 0; row < count; row++) {
3759
- if (column[row] >= 0) {
3900
+ if (column[row] >= 0 && column[row] < indexLimit) {
3760
3901
  needed.add(column[row]);
3761
3902
  }
3762
3903
  }
3904
+ if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
3905
+ counted = true;
3906
+ indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
3907
+ for (const index of needed) {
3908
+ if (index >= indexLimit) {
3909
+ needed.delete(index);
3910
+ }
3911
+ }
3912
+ }
3763
3913
  }
3764
3914
  this._emitProgress("symbol-analysis", position + 1, fields.length);
3765
3915
  });
3766
3916
  return symbolUsage;
3767
3917
  }
3918
+ /**
3919
+ * How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
3920
+ *
3921
+ * The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
3922
+ * `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
3923
+ * validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
3924
+ * same way wherever it is met.
3925
+ *
3926
+ * @param {any} field The field's header.
3927
+ * @return {number} Its symbols.
3928
+ * @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
3929
+ * @private
3930
+ */
3931
+ _countFieldSymbols(field) {
3932
+ assert3(
3933
+ this._buffer && this._symbolTableOffset && this._indexTableOffset,
3934
+ "The QVD file has not been read before its symbols were counted."
3935
+ );
3936
+ const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3937
+ validateFieldMetadata(field, symbolBuffer.length, this._path);
3938
+ const offset = headerInteger(field["Offset"]);
3939
+ return countFieldSymbols(
3940
+ symbolBuffer,
3941
+ offset,
3942
+ offset + headerInteger(field["Length"]),
3943
+ field["FieldName"],
3944
+ this._path
3945
+ );
3946
+ }
3768
3947
  /**
3769
3948
  * Parses the symbol table of the QVD file. This method is part of the parsing process
3770
3949
  * and should not be called directly.
@@ -3795,7 +3974,7 @@ var init_QvdFileReader = __esm({
3795
3974
  const fields = this._selectedFields;
3796
3975
  const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3797
3976
  const symbolTableSize = symbolBuffer.length;
3798
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3977
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3799
3978
  validateSymbolTableSize(symbolTableSize, this._path, totalRows);
3800
3979
  if (this._headerMatchesFile) {
3801
3980
  validateMemoryAvailability(
@@ -3813,10 +3992,11 @@ var init_QvdFileReader = __esm({
3813
3992
  for (const field of allFields) {
3814
3993
  validateFieldMetadata(field, symbolBuffer.length, this._path);
3815
3994
  }
3995
+ validateSymbolAreas(allFields, this._path);
3816
3996
  this._symbolTable = fields.map((field, position) => {
3817
3997
  this._throwIfAborted();
3818
- const symbolsOffset = parseInt(field["Offset"], 10);
3819
- const symbolsLength = parseInt(field["Length"], 10);
3998
+ const symbolsOffset = headerInteger(field["Offset"]);
3999
+ const symbolsLength = headerInteger(field["Length"]);
3820
4000
  const symbols = parseFieldSymbols(
3821
4001
  symbolBuffer,
3822
4002
  symbolsOffset,
@@ -3850,25 +4030,35 @@ var init_QvdFileReader = __esm({
3850
4030
  * `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
3851
4031
  * and how many iterations run. Nothing about the decoder changed to support it.
3852
4032
  *
4033
+ * Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
4034
+ * NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
4035
+ * are built, because every read decodes through this method and a columnar read hands its codes
4036
+ * straight to the caller. Rows outside the window are not decoded, so they are not checked.
4037
+ *
3853
4038
  * @param {QvdRowWindow} window The rows to decode.
4039
+ * @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
3854
4040
  */
3855
4041
  async _parseIndexTable(window) {
3856
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
3857
- this._rowsDecoded = rowsToLoad;
3858
- this._indexColumns = fields.map((field, position) => {
4042
+ const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
4043
+ assert3(this._symbolTable, "The QVD file symbol table has not been parsed.");
4044
+ const symbolTable = this._symbolTable;
4045
+ const columns = fields.map((field, position) => {
3859
4046
  this._throwIfAborted();
3860
4047
  const column = decodeIndexColumn(
3861
4048
  indexBuffer,
3862
4049
  recordSize,
3863
4050
  rowsToLoad,
3864
- parseInt(field["BitOffset"], 10),
3865
- parseInt(field["BitWidth"], 10),
3866
- parseInt(field["Bias"], 10),
3867
- new Int32Array(rowsToLoad)
4051
+ headerInteger(field["BitOffset"]),
4052
+ headerInteger(field["BitWidth"]),
4053
+ headerInteger(field["Bias"]),
4054
+ new Int32Array(rowsToLoad),
4055
+ { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
3868
4056
  );
3869
4057
  this._emitProgress("index-table", position + 1, fields.length);
3870
4058
  return column;
3871
4059
  });
4060
+ this._indexColumns = columns;
4061
+ this._rowsDecoded = rowsToLoad;
3872
4062
  }
3873
4063
  /**
3874
4064
  * Reads the file's schema and header metadata, without touching the symbol or index tables.
@@ -3890,23 +4080,13 @@ var init_QvdFileReader = __esm({
3890
4080
  await this._parseHeader();
3891
4081
  this._emitProgress("header", 1, 1);
3892
4082
  this._throwIfAborted();
3893
- assert3(this._header, "The QVD file header has not been parsed.");
4083
+ assert3(this._header && this._allFields, "The QVD file header has not been parsed.");
3894
4084
  const header = this._header["QvdTableHeader"];
3895
- let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
3896
- if (!Array.isArray(fields)) {
3897
- fields = [fields];
3898
- }
3899
- if (fields.length === 0) {
3900
- throw new QvdCorruptedError("The QVD file header declares no fields", {
3901
- file: this._path,
3902
- stage: "readMetadata"
3903
- });
3904
- }
3905
- const columns = fields.map((field) => field["FieldName"]);
3906
- const rowCount = parseInt(header["NoOfRecords"], 10);
4085
+ const columns = this._allFields.map((field) => field["FieldName"]);
4086
+ const rowCount = headerInteger(header["NoOfRecords"]);
3907
4087
  validateRecordCount(rowCount, this._path, "readMetadata");
3908
4088
  const shape = new QvdDataFrame([], columns, header, {
3909
- symbolTableBytes: parseInt(header["Offset"], 10),
4089
+ symbolTableBytes: headerInteger(header["Offset"]),
3910
4090
  totalRows: rowCount,
3911
4091
  rowsLoaded: 0,
3912
4092
  symbolFiltering: false,
@@ -3997,7 +4177,8 @@ var init_QvdFileReader = __esm({
3997
4177
  * `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
3998
4178
  *
3999
4179
  * A window covering no rows yields nothing at all, rather than one empty frame - so
4000
- * `for await` over an exhausted offset does nothing, which is what a paging loop wants.
4180
+ * `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
4181
+ * is still checked, as every read of a file's rows checks it.
4001
4182
  *
4002
4183
  * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
4003
4184
  * The rows to cover, in the same spellings `load()` accepts.
@@ -4015,6 +4196,10 @@ var init_QvdFileReader = __esm({
4015
4196
  const liveRows = { rows: chunkSize * 2, perChunk: 2 };
4016
4197
  const rows = normaliseWindow(window, this._path);
4017
4198
  const prepared = await this._prepare(rows, liveRows);
4199
+ if (prepared.rowsAvailable === 0) {
4200
+ this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
4201
+ return;
4202
+ }
4018
4203
  for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
4019
4204
  this._throwIfAborted();
4020
4205
  const count = Math.min(chunkSize, prepared.rowsAvailable - done);
@@ -4066,8 +4251,8 @@ var init_QvdFileReader = __esm({
4066
4251
  this._emitProgress("header", 1, 1);
4067
4252
  this._throwIfAborted();
4068
4253
  assert3(this._header, "The QVD file header has not been parsed.");
4069
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
4070
- const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
4254
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
4255
+ const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
4071
4256
  const resolved = resolveWindow(window, totalRows);
4072
4257
  const rowsAvailable = resolved.limit;
4073
4258
  let symbolsToKeep = null;