qvdjs 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -5,7 +5,7 @@ var path2 = require('path');
5
5
  var assert3 = require('assert');
6
6
  var crypto2 = require('crypto');
7
7
  var promises = require('timers/promises');
8
- var xml2 = require('xml2js');
8
+ var xml = require('xml2js');
9
9
  var os = require('os');
10
10
  var v8 = require('v8');
11
11
  var util = require('util');
@@ -16,7 +16,7 @@ var fs2__default = /*#__PURE__*/_interopDefault(fs2);
16
16
  var path2__default = /*#__PURE__*/_interopDefault(path2);
17
17
  var assert3__default = /*#__PURE__*/_interopDefault(assert3);
18
18
  var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
19
- var xml2__default = /*#__PURE__*/_interopDefault(xml2);
19
+ var xml__default = /*#__PURE__*/_interopDefault(xml);
20
20
  var os__default = /*#__PURE__*/_interopDefault(os);
21
21
  var v8__default = /*#__PURE__*/_interopDefault(v8);
22
22
 
@@ -1098,8 +1098,12 @@ function fieldGeometry(bitOffset, bitWidth) {
1098
1098
  byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
1099
1099
  };
1100
1100
  }
1101
- function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
1101
+ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
1102
+ const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
1102
1103
  if (bitWidth === 0) {
1104
+ if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
1105
+ refuse2(bounds, 0, bias, bias);
1106
+ }
1103
1107
  out.fill(bias, 0, rowCount);
1104
1108
  return out;
1105
1109
  }
@@ -1113,10 +1117,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
1113
1117
  if (byteCount > 2) acc += buffer[base + 2] * 65536;
1114
1118
  if (byteCount > 3) acc += buffer[base + 3] * 16777216;
1115
1119
  if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
1116
- out[row] = Math.floor(acc / divisor) % modulus + bias;
1120
+ const index = Math.floor(acc / divisor) % modulus + bias;
1121
+ if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
1122
+ refuse2(bounds, row, index, bias);
1123
+ }
1124
+ out[row] = index;
1117
1125
  }
1118
1126
  return out;
1119
1127
  }
1128
+ function refuse2(bounds, row, index, bias) {
1129
+ throw new exports.QvdCorruptedError("Symbol index out of range", {
1130
+ field: bounds.field,
1131
+ row: bounds.firstRow + row,
1132
+ symbolIndex: index,
1133
+ symbolCount: bounds.symbolCount,
1134
+ bias,
1135
+ file: bounds.file,
1136
+ stage: "parseIndexTable"
1137
+ });
1138
+ }
1120
1139
  function writeBitField(buffer, recordBase, geometry, value) {
1121
1140
  const { byteStart, shift, byteCount } = geometry;
1122
1141
  if (byteCount === 0) {
@@ -1133,10 +1152,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
1133
1152
  var MAX_BIT_WIDTH, POW2;
1134
1153
  var init_bitUtils = __esm({
1135
1154
  "src/util/bitUtils.js"() {
1155
+ init_QvdErrors();
1136
1156
  MAX_BIT_WIDTH = 31;
1137
1157
  POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
1138
1158
  __name(fieldGeometry, "fieldGeometry");
1139
1159
  __name(decodeIndexColumn, "decodeIndexColumn");
1160
+ __name(refuse2, "refuse");
1140
1161
  __name(writeBitField, "writeBitField");
1141
1162
  }
1142
1163
  });
@@ -1781,7 +1802,7 @@ var init_QvdFileWriter = __esm({
1781
1802
  });
1782
1803
  }
1783
1804
  }
1784
- const builder = new xml2__default.default.Builder({
1805
+ const builder = new xml__default.default.Builder({
1785
1806
  renderOpts: {
1786
1807
  pretty: true,
1787
1808
  newline: "\r\n",
@@ -2343,6 +2364,13 @@ var init_memoryUtils = __esm({
2343
2364
  });
2344
2365
 
2345
2366
  // src/util/validationUtils.js
2367
+ function headerInteger(value) {
2368
+ if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
2369
+ return NaN;
2370
+ }
2371
+ const number = Number(value);
2372
+ return number === 0 ? 0 : number;
2373
+ }
2346
2374
  function validateHeaderStructure(headerObj, filePath, stage) {
2347
2375
  const tableHeader = headerObj?.["QvdTableHeader"];
2348
2376
  if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
@@ -2352,7 +2380,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
2352
2380
  stage
2353
2381
  });
2354
2382
  }
2355
- const symbolTableLength = parseInt(tableHeader["Offset"], 10);
2383
+ const symbolTableLength = headerInteger(tableHeader["Offset"]);
2356
2384
  if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
2357
2385
  throw new exports.QvdCorruptedError("Invalid symbol table offset", {
2358
2386
  offset: tableHeader["Offset"],
@@ -2360,6 +2388,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
2360
2388
  stage
2361
2389
  });
2362
2390
  }
2391
+ const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
2392
+ const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
2393
+ if (fieldList.length === 0) {
2394
+ throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
2395
+ file: filePath,
2396
+ stage
2397
+ });
2398
+ }
2399
+ const malformedIndex = fieldList.findIndex(
2400
+ (field) => field === null || typeof field !== "object" || Array.isArray(field)
2401
+ );
2402
+ if (malformedIndex !== -1) {
2403
+ throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
2404
+ fieldIndex: malformedIndex,
2405
+ fieldCount: fieldList.length,
2406
+ file: filePath,
2407
+ stage
2408
+ });
2409
+ }
2410
+ const seen = /* @__PURE__ */ new Map();
2411
+ fieldList.forEach((field, fieldIndex) => {
2412
+ const name = field["FieldName"];
2413
+ if (typeof name !== "string" || name === "") {
2414
+ throw new exports.QvdCorruptedError("The QVD file header declares a field with no usable name", {
2415
+ fieldIndex,
2416
+ fieldName: name,
2417
+ file: filePath,
2418
+ stage
2419
+ });
2420
+ }
2421
+ if (seen.has(name)) {
2422
+ throw new exports.QvdCorruptedError("The QVD file header declares two fields with the same name", {
2423
+ field: name,
2424
+ fieldIndexes: [seen.get(name), fieldIndex],
2425
+ file: filePath,
2426
+ stage
2427
+ });
2428
+ }
2429
+ seen.set(name, fieldIndex);
2430
+ });
2431
+ return fieldList;
2363
2432
  }
2364
2433
  function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
2365
2434
  const heapLimit = getHeapLimit();
@@ -2405,8 +2474,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
2405
2474
  }
2406
2475
  }
2407
2476
  function validateFieldMetadata(field, symbolBufferLength, filePath) {
2408
- const symbolsOffset = parseInt(field["Offset"], 10);
2409
- const symbolsLength = parseInt(field["Length"], 10);
2477
+ const symbolsOffset = headerInteger(field["Offset"]);
2478
+ const symbolsLength = headerInteger(field["Length"]);
2410
2479
  if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
2411
2480
  throw new exports.QvdCorruptedError("Invalid symbol offset", {
2412
2481
  field: field["FieldName"],
@@ -2434,6 +2503,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
2434
2503
  });
2435
2504
  }
2436
2505
  }
2506
+ function firstOverlap(ranges) {
2507
+ const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
2508
+ let furthest = null;
2509
+ for (const range of claimed) {
2510
+ if (furthest !== null && range.start < furthest.start + furthest.length) {
2511
+ return [furthest, range];
2512
+ }
2513
+ if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
2514
+ furthest = range;
2515
+ }
2516
+ }
2517
+ return null;
2518
+ }
2519
+ function validateSymbolAreas(fields, filePath) {
2520
+ const overlap = firstOverlap(
2521
+ fields.map((field) => ({
2522
+ field: field["FieldName"],
2523
+ start: headerInteger(field["Offset"]),
2524
+ length: headerInteger(field["Length"])
2525
+ }))
2526
+ );
2527
+ if (overlap !== null) {
2528
+ const [first, second] = overlap;
2529
+ throw new exports.QvdCorruptedError("Symbol areas overlap", {
2530
+ field: second.field,
2531
+ offset: second.start,
2532
+ length: second.length,
2533
+ overlaps: first.field,
2534
+ overlapsOffset: first.start,
2535
+ overlapsLength: first.length,
2536
+ file: filePath,
2537
+ stage: "parseSymbolTable"
2538
+ });
2539
+ }
2540
+ }
2541
+ function validateBitFields(fields, filePath) {
2542
+ const overlap = firstOverlap(
2543
+ fields.map((field) => ({
2544
+ field: field["FieldName"],
2545
+ start: headerInteger(field["BitOffset"]),
2546
+ length: headerInteger(field["BitWidth"])
2547
+ }))
2548
+ );
2549
+ if (overlap !== null) {
2550
+ const [first, second] = overlap;
2551
+ throw new exports.QvdCorruptedError("Bit fields overlap", {
2552
+ field: second.field,
2553
+ bitOffset: second.start,
2554
+ bitWidth: second.length,
2555
+ overlaps: first.field,
2556
+ overlapsBitOffset: first.start,
2557
+ overlapsBitWidth: first.length,
2558
+ file: filePath,
2559
+ stage: "parseIndexTable"
2560
+ });
2561
+ }
2562
+ }
2437
2563
  function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
2438
2564
  if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
2439
2565
  throw new exports.QvdCorruptedError("Invalid number of records", {
@@ -2443,14 +2569,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
2443
2569
  });
2444
2570
  }
2445
2571
  }
2446
- function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
2572
+ function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
2447
2573
  if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
2448
2574
  throw new exports.QvdCorruptedError("Invalid record byte size", {
2449
2575
  recordSize,
2450
2576
  file: filePath,
2451
- stage: "parseIndexTable"
2577
+ stage
2452
2578
  });
2453
2579
  }
2580
+ }
2581
+ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
2582
+ validateRecordSize(recordSize, filePath);
2454
2583
  validateRecordCount(totalRows, filePath);
2455
2584
  if (recordSize === 0 && totalRows > 0) {
2456
2585
  throw new exports.QvdCorruptedError("Record byte size cannot be zero when records exist", {
@@ -2534,8 +2663,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
2534
2663
  }
2535
2664
  }
2536
2665
  function validateFieldBitMetadata(field, recordSize, filePath) {
2537
- const bitOffset = parseInt(field["BitOffset"], 10);
2538
- const bitWidth = parseInt(field["BitWidth"], 10);
2666
+ const bitOffset = headerInteger(field["BitOffset"]);
2667
+ const bitWidth = headerInteger(field["BitWidth"]);
2539
2668
  if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
2540
2669
  throw new exports.QvdCorruptedError("Invalid bit offset", {
2541
2670
  field: field["FieldName"],
@@ -2552,7 +2681,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
2552
2681
  stage: "parseIndexTable"
2553
2682
  });
2554
2683
  }
2555
- const bias = parseInt(field["Bias"], 10);
2684
+ const bias = headerInteger(field["Bias"]);
2556
2685
  if (isNaN(bias) || !Number.isSafeInteger(bias)) {
2557
2686
  throw new exports.QvdCorruptedError("Invalid bias", {
2558
2687
  field: field["FieldName"],
@@ -2570,6 +2699,23 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
2570
2699
  stage: "parseIndexTable"
2571
2700
  });
2572
2701
  }
2702
+ if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
2703
+ throw new exports.QvdCorruptedError("Bias out of range", {
2704
+ field: field["FieldName"],
2705
+ bias,
2706
+ bitWidth,
2707
+ file: filePath,
2708
+ stage: "parseIndexTable"
2709
+ });
2710
+ }
2711
+ if (bias !== 0 && bias !== -2) {
2712
+ throw new exports.QvdCorruptedError("Bias is neither 0 nor -2", {
2713
+ field: field["FieldName"],
2714
+ bias,
2715
+ file: filePath,
2716
+ stage: "parseIndexTable"
2717
+ });
2718
+ }
2573
2719
  const recordSizeInBits = recordSize * 8;
2574
2720
  if (bitOffset + bitWidth > recordSizeInBits) {
2575
2721
  throw new exports.QvdCorruptedError("Bit field extends beyond record size", {
@@ -2587,11 +2733,16 @@ var init_validationUtils = __esm({
2587
2733
  init_QvdErrors();
2588
2734
  init_memoryUtils();
2589
2735
  init_bitUtils();
2736
+ __name(headerInteger, "headerInteger");
2590
2737
  __name(validateHeaderStructure, "validateHeaderStructure");
2591
2738
  __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
2592
2739
  __name(validateSymbolTableSize, "validateSymbolTableSize");
2593
2740
  __name(validateFieldMetadata, "validateFieldMetadata");
2741
+ __name(firstOverlap, "firstOverlap");
2742
+ __name(validateSymbolAreas, "validateSymbolAreas");
2743
+ __name(validateBitFields, "validateBitFields");
2594
2744
  __name(validateRecordCount, "validateRecordCount");
2745
+ __name(validateRecordSize, "validateRecordSize");
2595
2746
  __name(validateIndexTableMetadata, "validateIndexTableMetadata");
2596
2747
  __name(validateFieldBitMetadata, "validateFieldBitMetadata");
2597
2748
  }
@@ -2697,7 +2848,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
2697
2848
  }
2698
2849
  return { numbers, texts };
2699
2850
  }
2700
- var MAX_TEXT_BYTES;
2851
+ function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
2852
+ return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
2853
+ }
2854
+ var MAX_TEXT_BYTES, DECODE_NOTHING;
2701
2855
  var init_symbolParser = __esm({
2702
2856
  "src/util/symbolParser.js"() {
2703
2857
  init_QvdErrors();
@@ -2705,6 +2859,8 @@ var init_symbolParser = __esm({
2705
2859
  __name(textEnd, "textEnd");
2706
2860
  __name(overflow, "overflow");
2707
2861
  __name(parseFieldSymbols, "parseFieldSymbols");
2862
+ DECODE_NOTHING = /* @__PURE__ */ new Set();
2863
+ __name(countFieldSymbols, "countFieldSymbols");
2708
2864
  }
2709
2865
  });
2710
2866
 
@@ -3218,7 +3374,32 @@ async function* chunksFrom(handle, chunkSize, failed) {
3218
3374
  position += bytesRead;
3219
3375
  }
3220
3376
  }
3221
- var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS; exports.QvdFileReader = void 0;
3377
+ async function parseHeaderXml(text, file, stage) {
3378
+ let parsed;
3379
+ try {
3380
+ parsed = await xml__default.default.parseStringPromise(text, { explicitArray: false });
3381
+ } catch (error) {
3382
+ throw new exports.QvdParseError(
3383
+ "The XML header could not be parsed.",
3384
+ {
3385
+ // The first line of the parser's message. The rest is the line and column it stopped at, which
3386
+ // stay on `cause`.
3387
+ reason: String(
3388
+ /** @type {any} */
3389
+ error?.message ?? error
3390
+ ).split("\n")[0],
3391
+ file,
3392
+ stage
3393
+ },
3394
+ { cause: error }
3395
+ );
3396
+ }
3397
+ if (!parsed) {
3398
+ throw new exports.QvdParseError("The XML header could not be parsed.", { file, stage });
3399
+ }
3400
+ return parsed;
3401
+ }
3402
+ var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
3222
3403
  var init_QvdFileReader = __esm({
3223
3404
  "src/QvdFileReader.js"() {
3224
3405
  init_QvdDataFrame();
@@ -3236,7 +3417,9 @@ var init_QvdFileReader = __esm({
3236
3417
  MAX_HEADER_SIZE = 16 * 1024 * 1024;
3237
3418
  READ_CHUNK_SIZE = 512 * 1024 * 1024;
3238
3419
  ANALYSIS_SLICE_ROWS = 65536;
3420
+ COUNT_SYMBOLS_PAST = 65536;
3239
3421
  __name(chunksFrom, "chunksFrom");
3422
+ __name(parseHeaderXml, "parseHeaderXml");
3240
3423
  exports.QvdFileReader = class {
3241
3424
  static {
3242
3425
  __name(this, "QvdFileReader");
@@ -3472,29 +3655,19 @@ var init_QvdFileReader = __esm({
3472
3655
  const headerBuffer = Buffer.concat(headerChunks);
3473
3656
  const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
3474
3657
  const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
3475
- const headerObj = await xml2__default.default.parseStringPromise(headerXml, { explicitArray: false });
3476
- if (!headerObj) {
3477
- throw new exports.QvdParseError("The XML header could not be parsed.", {
3478
- file: this._path,
3479
- stage: "readData"
3480
- });
3481
- }
3482
- validateHeaderStructure(headerObj, this._path, "readData");
3658
+ const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
3659
+ const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
3483
3660
  const symbolTableOffset = headerEndIndex;
3484
- const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
3661
+ const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
3485
3662
  const indexTableOffset = symbolTableOffset + symbolTableLength;
3486
- const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
3487
- const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
3663
+ const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
3664
+ const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
3488
3665
  if (headerOnly) {
3489
3666
  this._buffer = headerBuffer.subarray(0, headerEndIndex);
3490
3667
  this._emitProgress("read", 1, 1);
3491
3668
  return;
3492
3669
  }
3493
- let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
3494
- if (headerFields && !Array.isArray(headerFields)) {
3495
- headerFields = [headerFields];
3496
- }
3497
- const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
3670
+ const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
3498
3671
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
3499
3672
  (value) => Number.isSafeInteger(value) && value >= 0
3500
3673
  );
@@ -3526,20 +3699,8 @@ var init_QvdFileReader = __esm({
3526
3699
  }
3527
3700
  const rowsToLoad = windowRows;
3528
3701
  validateSymbolTableSizeEarly(symbolTableLength, this._path);
3529
- for (const [name, value] of [
3530
- ["Offset", symbolTableLength],
3531
- ["RecordByteSize", recordSize],
3532
- ["NoOfRecords", totalRows]
3533
- ]) {
3534
- if (!Number.isSafeInteger(value) || Number(value) < 0) {
3535
- throw new exports.QvdCorruptedError(`Invalid header value: ${name}`, {
3536
- name,
3537
- value,
3538
- file: this._path,
3539
- stage: "readData"
3540
- });
3541
- }
3542
- }
3702
+ validateRecordSize(recordSize, this._path, "readData");
3703
+ validateRecordCount(totalRows, this._path, "readData");
3543
3704
  const skippedIndexBytes = resolved.offset * recordSize;
3544
3705
  const indexTableBytesToRead = rowsToLoad * recordSize;
3545
3706
  const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
@@ -3637,36 +3798,12 @@ var init_QvdFileReader = __esm({
3637
3798
  }
3638
3799
  const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
3639
3800
  const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
3640
- this._header = await xml2__default.default.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
3641
- if (!this._header) {
3642
- throw new exports.QvdParseError("The XML header could not be parsed.", {
3643
- file: this._path,
3644
- stage: "parseHeader"
3645
- });
3646
- }
3647
- validateHeaderStructure(this._header, this._path, "parseHeader");
3648
- const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
3649
- const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
3650
- if (fieldList.length === 0) {
3651
- throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
3652
- file: this._path,
3653
- stage: "parseHeader"
3654
- });
3655
- }
3656
- const malformedIndex = fieldList.findIndex(
3657
- (field) => field === null || typeof field !== "object" || Array.isArray(field)
3658
- );
3659
- if (malformedIndex !== -1) {
3660
- throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
3661
- fieldIndex: malformedIndex,
3662
- fieldCount: fieldList.length,
3663
- file: this._path,
3664
- stage: "parseHeader"
3665
- });
3666
- }
3801
+ this._fieldBitMetadataValidated = false;
3802
+ this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
3803
+ const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
3667
3804
  this._headerOffset = headerBeginIndex;
3668
3805
  this._symbolTableOffset = headerEndIndex;
3669
- this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3806
+ this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
3670
3807
  this._allFields = fieldList;
3671
3808
  this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
3672
3809
  }
@@ -3682,8 +3819,8 @@ var init_QvdFileReader = __esm({
3682
3819
  * @param {QvdRowWindow} window The rows of interest, as file row indices.
3683
3820
  * @param {string} stage Stage name for any error raised here.
3684
3821
  * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
3685
- * indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
3686
- * record, so the decoder always counts from zero.
3822
+ * indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
3823
+ * window's first record, file row `firstRow`, so the decoder always counts from zero.
3687
3824
  * @private
3688
3825
  */
3689
3826
  _planIndexTable(window, stage) {
@@ -3698,9 +3835,9 @@ var init_QvdFileReader = __esm({
3698
3835
  }
3699
3836
  const allFields = this._allFields;
3700
3837
  const fields = this._selectedFields;
3701
- const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
3702
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3703
- const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
3838
+ const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
3839
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3840
+ const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
3704
3841
  const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
3705
3842
  validateIndexTableMetadata(
3706
3843
  recordSize,
@@ -3723,13 +3860,14 @@ var init_QvdFileReader = __esm({
3723
3860
  for (const field of allFields) {
3724
3861
  validateFieldBitMetadata(field, recordSize, this._path);
3725
3862
  }
3863
+ validateBitFields(allFields, this._path);
3726
3864
  this._fieldBitMetadataValidated = true;
3727
3865
  }
3728
3866
  assert3__default.default(
3729
3867
  rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
3730
3868
  `The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
3731
3869
  );
3732
- return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
3870
+ return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
3733
3871
  }
3734
3872
  /**
3735
3873
  * Analyzes the index table to determine which symbols are actually needed.
@@ -3753,9 +3891,12 @@ var init_QvdFileReader = __esm({
3753
3891
  this._throwIfAborted();
3754
3892
  const needed = /* @__PURE__ */ new Set();
3755
3893
  symbolUsage[position] = needed;
3756
- const bitOffset = parseInt(field["BitOffset"], 10);
3757
- const bitWidth = parseInt(field["BitWidth"], 10);
3758
- const bias = parseInt(field["Bias"], 10);
3894
+ const bitOffset = headerInteger(field["BitOffset"]);
3895
+ const bitWidth = headerInteger(field["BitWidth"]);
3896
+ const bias = headerInteger(field["Bias"]);
3897
+ const length = headerInteger(field["Length"]);
3898
+ let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
3899
+ let counted = false;
3759
3900
  for (let first = 0; first < rowsToLoad; first += sliceRows) {
3760
3901
  const count = Math.min(sliceRows, rowsToLoad - first);
3761
3902
  decodeIndexColumn(
@@ -3768,15 +3909,53 @@ var init_QvdFileReader = __esm({
3768
3909
  column
3769
3910
  );
3770
3911
  for (let row = 0; row < count; row++) {
3771
- if (column[row] >= 0) {
3912
+ if (column[row] >= 0 && column[row] < indexLimit) {
3772
3913
  needed.add(column[row]);
3773
3914
  }
3774
3915
  }
3916
+ if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
3917
+ counted = true;
3918
+ indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
3919
+ for (const index of needed) {
3920
+ if (index >= indexLimit) {
3921
+ needed.delete(index);
3922
+ }
3923
+ }
3924
+ }
3775
3925
  }
3776
3926
  this._emitProgress("symbol-analysis", position + 1, fields.length);
3777
3927
  });
3778
3928
  return symbolUsage;
3779
3929
  }
3930
+ /**
3931
+ * How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
3932
+ *
3933
+ * The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
3934
+ * `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
3935
+ * validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
3936
+ * same way wherever it is met.
3937
+ *
3938
+ * @param {any} field The field's header.
3939
+ * @return {number} Its symbols.
3940
+ * @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
3941
+ * @private
3942
+ */
3943
+ _countFieldSymbols(field) {
3944
+ assert3__default.default(
3945
+ this._buffer && this._symbolTableOffset && this._indexTableOffset,
3946
+ "The QVD file has not been read before its symbols were counted."
3947
+ );
3948
+ const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3949
+ validateFieldMetadata(field, symbolBuffer.length, this._path);
3950
+ const offset = headerInteger(field["Offset"]);
3951
+ return countFieldSymbols(
3952
+ symbolBuffer,
3953
+ offset,
3954
+ offset + headerInteger(field["Length"]),
3955
+ field["FieldName"],
3956
+ this._path
3957
+ );
3958
+ }
3780
3959
  /**
3781
3960
  * Parses the symbol table of the QVD file. This method is part of the parsing process
3782
3961
  * and should not be called directly.
@@ -3807,7 +3986,7 @@ var init_QvdFileReader = __esm({
3807
3986
  const fields = this._selectedFields;
3808
3987
  const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3809
3988
  const symbolTableSize = symbolBuffer.length;
3810
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3989
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3811
3990
  validateSymbolTableSize(symbolTableSize, this._path, totalRows);
3812
3991
  if (this._headerMatchesFile) {
3813
3992
  validateMemoryAvailability(
@@ -3825,10 +4004,11 @@ var init_QvdFileReader = __esm({
3825
4004
  for (const field of allFields) {
3826
4005
  validateFieldMetadata(field, symbolBuffer.length, this._path);
3827
4006
  }
4007
+ validateSymbolAreas(allFields, this._path);
3828
4008
  this._symbolTable = fields.map((field, position) => {
3829
4009
  this._throwIfAborted();
3830
- const symbolsOffset = parseInt(field["Offset"], 10);
3831
- const symbolsLength = parseInt(field["Length"], 10);
4010
+ const symbolsOffset = headerInteger(field["Offset"]);
4011
+ const symbolsLength = headerInteger(field["Length"]);
3832
4012
  const symbols = parseFieldSymbols(
3833
4013
  symbolBuffer,
3834
4014
  symbolsOffset,
@@ -3862,25 +4042,35 @@ var init_QvdFileReader = __esm({
3862
4042
  * `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
3863
4043
  * and how many iterations run. Nothing about the decoder changed to support it.
3864
4044
  *
4045
+ * Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
4046
+ * NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
4047
+ * are built, because every read decodes through this method and a columnar read hands its codes
4048
+ * straight to the caller. Rows outside the window are not decoded, so they are not checked.
4049
+ *
3865
4050
  * @param {QvdRowWindow} window The rows to decode.
4051
+ * @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
3866
4052
  */
3867
4053
  async _parseIndexTable(window) {
3868
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
3869
- this._rowsDecoded = rowsToLoad;
3870
- this._indexColumns = fields.map((field, position) => {
4054
+ const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
4055
+ assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
4056
+ const symbolTable = this._symbolTable;
4057
+ const columns = fields.map((field, position) => {
3871
4058
  this._throwIfAborted();
3872
4059
  const column = decodeIndexColumn(
3873
4060
  indexBuffer,
3874
4061
  recordSize,
3875
4062
  rowsToLoad,
3876
- parseInt(field["BitOffset"], 10),
3877
- parseInt(field["BitWidth"], 10),
3878
- parseInt(field["Bias"], 10),
3879
- new Int32Array(rowsToLoad)
4063
+ headerInteger(field["BitOffset"]),
4064
+ headerInteger(field["BitWidth"]),
4065
+ headerInteger(field["Bias"]),
4066
+ new Int32Array(rowsToLoad),
4067
+ { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
3880
4068
  );
3881
4069
  this._emitProgress("index-table", position + 1, fields.length);
3882
4070
  return column;
3883
4071
  });
4072
+ this._indexColumns = columns;
4073
+ this._rowsDecoded = rowsToLoad;
3884
4074
  }
3885
4075
  /**
3886
4076
  * Reads the file's schema and header metadata, without touching the symbol or index tables.
@@ -3902,23 +4092,13 @@ var init_QvdFileReader = __esm({
3902
4092
  await this._parseHeader();
3903
4093
  this._emitProgress("header", 1, 1);
3904
4094
  this._throwIfAborted();
3905
- assert3__default.default(this._header, "The QVD file header has not been parsed.");
4095
+ assert3__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
3906
4096
  const header = this._header["QvdTableHeader"];
3907
- let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
3908
- if (!Array.isArray(fields)) {
3909
- fields = [fields];
3910
- }
3911
- if (fields.length === 0) {
3912
- throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
3913
- file: this._path,
3914
- stage: "readMetadata"
3915
- });
3916
- }
3917
- const columns = fields.map((field) => field["FieldName"]);
3918
- const rowCount = parseInt(header["NoOfRecords"], 10);
4097
+ const columns = this._allFields.map((field) => field["FieldName"]);
4098
+ const rowCount = headerInteger(header["NoOfRecords"]);
3919
4099
  validateRecordCount(rowCount, this._path, "readMetadata");
3920
4100
  const shape = new exports.QvdDataFrame([], columns, header, {
3921
- symbolTableBytes: parseInt(header["Offset"], 10),
4101
+ symbolTableBytes: headerInteger(header["Offset"]),
3922
4102
  totalRows: rowCount,
3923
4103
  rowsLoaded: 0,
3924
4104
  symbolFiltering: false,
@@ -4009,7 +4189,8 @@ var init_QvdFileReader = __esm({
4009
4189
  * `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
4010
4190
  *
4011
4191
  * A window covering no rows yields nothing at all, rather than one empty frame - so
4012
- * `for await` over an exhausted offset does nothing, which is what a paging loop wants.
4192
+ * `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
4193
+ * is still checked, as every read of a file's rows checks it.
4013
4194
  *
4014
4195
  * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
4015
4196
  * The rows to cover, in the same spellings `load()` accepts.
@@ -4027,6 +4208,10 @@ var init_QvdFileReader = __esm({
4027
4208
  const liveRows = { rows: chunkSize * 2, perChunk: 2 };
4028
4209
  const rows = normaliseWindow(window, this._path);
4029
4210
  const prepared = await this._prepare(rows, liveRows);
4211
+ if (prepared.rowsAvailable === 0) {
4212
+ this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
4213
+ return;
4214
+ }
4030
4215
  for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
4031
4216
  this._throwIfAborted();
4032
4217
  const count = Math.min(chunkSize, prepared.rowsAvailable - done);
@@ -4078,8 +4263,8 @@ var init_QvdFileReader = __esm({
4078
4263
  this._emitProgress("header", 1, 1);
4079
4264
  this._throwIfAborted();
4080
4265
  assert3__default.default(this._header, "The QVD file header has not been parsed.");
4081
- const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
4082
- const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
4266
+ const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
4267
+ const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
4083
4268
  const resolved = resolveWindow(window, totalRows);
4084
4269
  const rowsAvailable = resolved.limit;
4085
4270
  let symbolsToKeep = null;