qvdjs 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/dist/index.cjs +295 -110
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +294 -109
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@ import path2 from 'path';
|
|
|
3
3
|
import assert3 from 'assert';
|
|
4
4
|
import crypto2 from 'crypto';
|
|
5
5
|
import { setTimeout } from 'timers/promises';
|
|
6
|
-
import
|
|
6
|
+
import xml from 'xml2js';
|
|
7
7
|
import os from 'os';
|
|
8
8
|
import v8 from 'v8';
|
|
9
9
|
import { types } from 'util';
|
|
@@ -1086,8 +1086,12 @@ function fieldGeometry(bitOffset, bitWidth) {
|
|
|
1086
1086
|
byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
|
|
1087
1087
|
};
|
|
1088
1088
|
}
|
|
1089
|
-
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
|
|
1089
|
+
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
|
|
1090
|
+
const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
|
|
1090
1091
|
if (bitWidth === 0) {
|
|
1092
|
+
if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
|
|
1093
|
+
refuse2(bounds, 0, bias, bias);
|
|
1094
|
+
}
|
|
1091
1095
|
out.fill(bias, 0, rowCount);
|
|
1092
1096
|
return out;
|
|
1093
1097
|
}
|
|
@@ -1101,10 +1105,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
|
|
|
1101
1105
|
if (byteCount > 2) acc += buffer[base + 2] * 65536;
|
|
1102
1106
|
if (byteCount > 3) acc += buffer[base + 3] * 16777216;
|
|
1103
1107
|
if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
|
|
1104
|
-
|
|
1108
|
+
const index = Math.floor(acc / divisor) % modulus + bias;
|
|
1109
|
+
if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
|
|
1110
|
+
refuse2(bounds, row, index, bias);
|
|
1111
|
+
}
|
|
1112
|
+
out[row] = index;
|
|
1105
1113
|
}
|
|
1106
1114
|
return out;
|
|
1107
1115
|
}
|
|
1116
|
+
function refuse2(bounds, row, index, bias) {
|
|
1117
|
+
throw new QvdCorruptedError("Symbol index out of range", {
|
|
1118
|
+
field: bounds.field,
|
|
1119
|
+
row: bounds.firstRow + row,
|
|
1120
|
+
symbolIndex: index,
|
|
1121
|
+
symbolCount: bounds.symbolCount,
|
|
1122
|
+
bias,
|
|
1123
|
+
file: bounds.file,
|
|
1124
|
+
stage: "parseIndexTable"
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1108
1127
|
function writeBitField(buffer, recordBase, geometry, value) {
|
|
1109
1128
|
const { byteStart, shift, byteCount } = geometry;
|
|
1110
1129
|
if (byteCount === 0) {
|
|
@@ -1121,10 +1140,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
|
|
|
1121
1140
|
var MAX_BIT_WIDTH, POW2;
|
|
1122
1141
|
var init_bitUtils = __esm({
|
|
1123
1142
|
"src/util/bitUtils.js"() {
|
|
1143
|
+
init_QvdErrors();
|
|
1124
1144
|
MAX_BIT_WIDTH = 31;
|
|
1125
1145
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
1126
1146
|
__name(fieldGeometry, "fieldGeometry");
|
|
1127
1147
|
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
1148
|
+
__name(refuse2, "refuse");
|
|
1128
1149
|
__name(writeBitField, "writeBitField");
|
|
1129
1150
|
}
|
|
1130
1151
|
});
|
|
@@ -1769,7 +1790,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1769
1790
|
});
|
|
1770
1791
|
}
|
|
1771
1792
|
}
|
|
1772
|
-
const builder = new
|
|
1793
|
+
const builder = new xml.Builder({
|
|
1773
1794
|
renderOpts: {
|
|
1774
1795
|
pretty: true,
|
|
1775
1796
|
newline: "\r\n",
|
|
@@ -2331,6 +2352,13 @@ var init_memoryUtils = __esm({
|
|
|
2331
2352
|
});
|
|
2332
2353
|
|
|
2333
2354
|
// src/util/validationUtils.js
|
|
2355
|
+
function headerInteger(value) {
|
|
2356
|
+
if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
|
|
2357
|
+
return NaN;
|
|
2358
|
+
}
|
|
2359
|
+
const number = Number(value);
|
|
2360
|
+
return number === 0 ? 0 : number;
|
|
2361
|
+
}
|
|
2334
2362
|
function validateHeaderStructure(headerObj, filePath, stage) {
|
|
2335
2363
|
const tableHeader = headerObj?.["QvdTableHeader"];
|
|
2336
2364
|
if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
|
|
@@ -2340,7 +2368,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
|
|
|
2340
2368
|
stage
|
|
2341
2369
|
});
|
|
2342
2370
|
}
|
|
2343
|
-
const symbolTableLength =
|
|
2371
|
+
const symbolTableLength = headerInteger(tableHeader["Offset"]);
|
|
2344
2372
|
if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
|
|
2345
2373
|
throw new QvdCorruptedError("Invalid symbol table offset", {
|
|
2346
2374
|
offset: tableHeader["Offset"],
|
|
@@ -2348,6 +2376,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
|
|
|
2348
2376
|
stage
|
|
2349
2377
|
});
|
|
2350
2378
|
}
|
|
2379
|
+
const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
|
|
2380
|
+
const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
|
|
2381
|
+
if (fieldList.length === 0) {
|
|
2382
|
+
throw new QvdCorruptedError("The QVD file header declares no fields", {
|
|
2383
|
+
file: filePath,
|
|
2384
|
+
stage
|
|
2385
|
+
});
|
|
2386
|
+
}
|
|
2387
|
+
const malformedIndex = fieldList.findIndex(
|
|
2388
|
+
(field) => field === null || typeof field !== "object" || Array.isArray(field)
|
|
2389
|
+
);
|
|
2390
|
+
if (malformedIndex !== -1) {
|
|
2391
|
+
throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
|
|
2392
|
+
fieldIndex: malformedIndex,
|
|
2393
|
+
fieldCount: fieldList.length,
|
|
2394
|
+
file: filePath,
|
|
2395
|
+
stage
|
|
2396
|
+
});
|
|
2397
|
+
}
|
|
2398
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2399
|
+
fieldList.forEach((field, fieldIndex) => {
|
|
2400
|
+
const name = field["FieldName"];
|
|
2401
|
+
if (typeof name !== "string" || name === "") {
|
|
2402
|
+
throw new QvdCorruptedError("The QVD file header declares a field with no usable name", {
|
|
2403
|
+
fieldIndex,
|
|
2404
|
+
fieldName: name,
|
|
2405
|
+
file: filePath,
|
|
2406
|
+
stage
|
|
2407
|
+
});
|
|
2408
|
+
}
|
|
2409
|
+
if (seen.has(name)) {
|
|
2410
|
+
throw new QvdCorruptedError("The QVD file header declares two fields with the same name", {
|
|
2411
|
+
field: name,
|
|
2412
|
+
fieldIndexes: [seen.get(name), fieldIndex],
|
|
2413
|
+
file: filePath,
|
|
2414
|
+
stage
|
|
2415
|
+
});
|
|
2416
|
+
}
|
|
2417
|
+
seen.set(name, fieldIndex);
|
|
2418
|
+
});
|
|
2419
|
+
return fieldList;
|
|
2351
2420
|
}
|
|
2352
2421
|
function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
|
|
2353
2422
|
const heapLimit = getHeapLimit();
|
|
@@ -2393,8 +2462,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
|
|
|
2393
2462
|
}
|
|
2394
2463
|
}
|
|
2395
2464
|
function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
2396
|
-
const symbolsOffset =
|
|
2397
|
-
const symbolsLength =
|
|
2465
|
+
const symbolsOffset = headerInteger(field["Offset"]);
|
|
2466
|
+
const symbolsLength = headerInteger(field["Length"]);
|
|
2398
2467
|
if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
|
|
2399
2468
|
throw new QvdCorruptedError("Invalid symbol offset", {
|
|
2400
2469
|
field: field["FieldName"],
|
|
@@ -2422,6 +2491,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
|
2422
2491
|
});
|
|
2423
2492
|
}
|
|
2424
2493
|
}
|
|
2494
|
+
function firstOverlap(ranges) {
|
|
2495
|
+
const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
|
|
2496
|
+
let furthest = null;
|
|
2497
|
+
for (const range of claimed) {
|
|
2498
|
+
if (furthest !== null && range.start < furthest.start + furthest.length) {
|
|
2499
|
+
return [furthest, range];
|
|
2500
|
+
}
|
|
2501
|
+
if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
|
|
2502
|
+
furthest = range;
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
return null;
|
|
2506
|
+
}
|
|
2507
|
+
function validateSymbolAreas(fields, filePath) {
|
|
2508
|
+
const overlap = firstOverlap(
|
|
2509
|
+
fields.map((field) => ({
|
|
2510
|
+
field: field["FieldName"],
|
|
2511
|
+
start: headerInteger(field["Offset"]),
|
|
2512
|
+
length: headerInteger(field["Length"])
|
|
2513
|
+
}))
|
|
2514
|
+
);
|
|
2515
|
+
if (overlap !== null) {
|
|
2516
|
+
const [first, second] = overlap;
|
|
2517
|
+
throw new QvdCorruptedError("Symbol areas overlap", {
|
|
2518
|
+
field: second.field,
|
|
2519
|
+
offset: second.start,
|
|
2520
|
+
length: second.length,
|
|
2521
|
+
overlaps: first.field,
|
|
2522
|
+
overlapsOffset: first.start,
|
|
2523
|
+
overlapsLength: first.length,
|
|
2524
|
+
file: filePath,
|
|
2525
|
+
stage: "parseSymbolTable"
|
|
2526
|
+
});
|
|
2527
|
+
}
|
|
2528
|
+
}
|
|
2529
|
+
function validateBitFields(fields, filePath) {
|
|
2530
|
+
const overlap = firstOverlap(
|
|
2531
|
+
fields.map((field) => ({
|
|
2532
|
+
field: field["FieldName"],
|
|
2533
|
+
start: headerInteger(field["BitOffset"]),
|
|
2534
|
+
length: headerInteger(field["BitWidth"])
|
|
2535
|
+
}))
|
|
2536
|
+
);
|
|
2537
|
+
if (overlap !== null) {
|
|
2538
|
+
const [first, second] = overlap;
|
|
2539
|
+
throw new QvdCorruptedError("Bit fields overlap", {
|
|
2540
|
+
field: second.field,
|
|
2541
|
+
bitOffset: second.start,
|
|
2542
|
+
bitWidth: second.length,
|
|
2543
|
+
overlaps: first.field,
|
|
2544
|
+
overlapsBitOffset: first.start,
|
|
2545
|
+
overlapsBitWidth: first.length,
|
|
2546
|
+
file: filePath,
|
|
2547
|
+
stage: "parseIndexTable"
|
|
2548
|
+
});
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2425
2551
|
function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
2426
2552
|
if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
|
|
2427
2553
|
throw new QvdCorruptedError("Invalid number of records", {
|
|
@@ -2431,14 +2557,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
|
2431
2557
|
});
|
|
2432
2558
|
}
|
|
2433
2559
|
}
|
|
2434
|
-
function
|
|
2560
|
+
function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
|
|
2435
2561
|
if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
|
|
2436
2562
|
throw new QvdCorruptedError("Invalid record byte size", {
|
|
2437
2563
|
recordSize,
|
|
2438
2564
|
file: filePath,
|
|
2439
|
-
stage
|
|
2565
|
+
stage
|
|
2440
2566
|
});
|
|
2441
2567
|
}
|
|
2568
|
+
}
|
|
2569
|
+
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
|
|
2570
|
+
validateRecordSize(recordSize, filePath);
|
|
2442
2571
|
validateRecordCount(totalRows, filePath);
|
|
2443
2572
|
if (recordSize === 0 && totalRows > 0) {
|
|
2444
2573
|
throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
|
|
@@ -2522,8 +2651,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
|
|
|
2522
2651
|
}
|
|
2523
2652
|
}
|
|
2524
2653
|
function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
2525
|
-
const bitOffset =
|
|
2526
|
-
const bitWidth =
|
|
2654
|
+
const bitOffset = headerInteger(field["BitOffset"]);
|
|
2655
|
+
const bitWidth = headerInteger(field["BitWidth"]);
|
|
2527
2656
|
if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
|
|
2528
2657
|
throw new QvdCorruptedError("Invalid bit offset", {
|
|
2529
2658
|
field: field["FieldName"],
|
|
@@ -2540,7 +2669,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2540
2669
|
stage: "parseIndexTable"
|
|
2541
2670
|
});
|
|
2542
2671
|
}
|
|
2543
|
-
const bias =
|
|
2672
|
+
const bias = headerInteger(field["Bias"]);
|
|
2544
2673
|
if (isNaN(bias) || !Number.isSafeInteger(bias)) {
|
|
2545
2674
|
throw new QvdCorruptedError("Invalid bias", {
|
|
2546
2675
|
field: field["FieldName"],
|
|
@@ -2558,6 +2687,23 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2558
2687
|
stage: "parseIndexTable"
|
|
2559
2688
|
});
|
|
2560
2689
|
}
|
|
2690
|
+
if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
|
|
2691
|
+
throw new QvdCorruptedError("Bias out of range", {
|
|
2692
|
+
field: field["FieldName"],
|
|
2693
|
+
bias,
|
|
2694
|
+
bitWidth,
|
|
2695
|
+
file: filePath,
|
|
2696
|
+
stage: "parseIndexTable"
|
|
2697
|
+
});
|
|
2698
|
+
}
|
|
2699
|
+
if (bias !== 0 && bias !== -2) {
|
|
2700
|
+
throw new QvdCorruptedError("Bias is neither 0 nor -2", {
|
|
2701
|
+
field: field["FieldName"],
|
|
2702
|
+
bias,
|
|
2703
|
+
file: filePath,
|
|
2704
|
+
stage: "parseIndexTable"
|
|
2705
|
+
});
|
|
2706
|
+
}
|
|
2561
2707
|
const recordSizeInBits = recordSize * 8;
|
|
2562
2708
|
if (bitOffset + bitWidth > recordSizeInBits) {
|
|
2563
2709
|
throw new QvdCorruptedError("Bit field extends beyond record size", {
|
|
@@ -2575,11 +2721,16 @@ var init_validationUtils = __esm({
|
|
|
2575
2721
|
init_QvdErrors();
|
|
2576
2722
|
init_memoryUtils();
|
|
2577
2723
|
init_bitUtils();
|
|
2724
|
+
__name(headerInteger, "headerInteger");
|
|
2578
2725
|
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2579
2726
|
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2580
2727
|
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2581
2728
|
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2729
|
+
__name(firstOverlap, "firstOverlap");
|
|
2730
|
+
__name(validateSymbolAreas, "validateSymbolAreas");
|
|
2731
|
+
__name(validateBitFields, "validateBitFields");
|
|
2582
2732
|
__name(validateRecordCount, "validateRecordCount");
|
|
2733
|
+
__name(validateRecordSize, "validateRecordSize");
|
|
2583
2734
|
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2584
2735
|
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
2585
2736
|
}
|
|
@@ -2685,7 +2836,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2685
2836
|
}
|
|
2686
2837
|
return { numbers, texts };
|
|
2687
2838
|
}
|
|
2688
|
-
|
|
2839
|
+
function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
|
|
2840
|
+
return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
|
|
2841
|
+
}
|
|
2842
|
+
var MAX_TEXT_BYTES, DECODE_NOTHING;
|
|
2689
2843
|
var init_symbolParser = __esm({
|
|
2690
2844
|
"src/util/symbolParser.js"() {
|
|
2691
2845
|
init_QvdErrors();
|
|
@@ -2693,6 +2847,8 @@ var init_symbolParser = __esm({
|
|
|
2693
2847
|
__name(textEnd, "textEnd");
|
|
2694
2848
|
__name(overflow, "overflow");
|
|
2695
2849
|
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2850
|
+
DECODE_NOTHING = /* @__PURE__ */ new Set();
|
|
2851
|
+
__name(countFieldSymbols, "countFieldSymbols");
|
|
2696
2852
|
}
|
|
2697
2853
|
});
|
|
2698
2854
|
|
|
@@ -3206,7 +3362,32 @@ async function* chunksFrom(handle, chunkSize, failed) {
|
|
|
3206
3362
|
position += bytesRead;
|
|
3207
3363
|
}
|
|
3208
3364
|
}
|
|
3209
|
-
|
|
3365
|
+
async function parseHeaderXml(text, file, stage) {
|
|
3366
|
+
let parsed;
|
|
3367
|
+
try {
|
|
3368
|
+
parsed = await xml.parseStringPromise(text, { explicitArray: false });
|
|
3369
|
+
} catch (error) {
|
|
3370
|
+
throw new QvdParseError(
|
|
3371
|
+
"The XML header could not be parsed.",
|
|
3372
|
+
{
|
|
3373
|
+
// The first line of the parser's message. The rest is the line and column it stopped at, which
|
|
3374
|
+
// stay on `cause`.
|
|
3375
|
+
reason: String(
|
|
3376
|
+
/** @type {any} */
|
|
3377
|
+
error?.message ?? error
|
|
3378
|
+
).split("\n")[0],
|
|
3379
|
+
file,
|
|
3380
|
+
stage
|
|
3381
|
+
},
|
|
3382
|
+
{ cause: error }
|
|
3383
|
+
);
|
|
3384
|
+
}
|
|
3385
|
+
if (!parsed) {
|
|
3386
|
+
throw new QvdParseError("The XML header could not be parsed.", { file, stage });
|
|
3387
|
+
}
|
|
3388
|
+
return parsed;
|
|
3389
|
+
}
|
|
3390
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST, QvdFileReader;
|
|
3210
3391
|
var init_QvdFileReader = __esm({
|
|
3211
3392
|
"src/QvdFileReader.js"() {
|
|
3212
3393
|
init_QvdDataFrame();
|
|
@@ -3224,7 +3405,9 @@ var init_QvdFileReader = __esm({
|
|
|
3224
3405
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
3225
3406
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
3226
3407
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
3408
|
+
COUNT_SYMBOLS_PAST = 65536;
|
|
3227
3409
|
__name(chunksFrom, "chunksFrom");
|
|
3410
|
+
__name(parseHeaderXml, "parseHeaderXml");
|
|
3228
3411
|
QvdFileReader = class {
|
|
3229
3412
|
static {
|
|
3230
3413
|
__name(this, "QvdFileReader");
|
|
@@ -3460,29 +3643,19 @@ var init_QvdFileReader = __esm({
|
|
|
3460
3643
|
const headerBuffer = Buffer.concat(headerChunks);
|
|
3461
3644
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3462
3645
|
const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
|
|
3463
|
-
const headerObj = await
|
|
3464
|
-
|
|
3465
|
-
throw new QvdParseError("The XML header could not be parsed.", {
|
|
3466
|
-
file: this._path,
|
|
3467
|
-
stage: "readData"
|
|
3468
|
-
});
|
|
3469
|
-
}
|
|
3470
|
-
validateHeaderStructure(headerObj, this._path, "readData");
|
|
3646
|
+
const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
|
|
3647
|
+
const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
|
|
3471
3648
|
const symbolTableOffset = headerEndIndex;
|
|
3472
|
-
const symbolTableLength =
|
|
3649
|
+
const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
|
|
3473
3650
|
const indexTableOffset = symbolTableOffset + symbolTableLength;
|
|
3474
|
-
const recordSize =
|
|
3475
|
-
const totalRows =
|
|
3651
|
+
const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
|
|
3652
|
+
const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
|
|
3476
3653
|
if (headerOnly) {
|
|
3477
3654
|
this._buffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3478
3655
|
this._emitProgress("read", 1, 1);
|
|
3479
3656
|
return;
|
|
3480
3657
|
}
|
|
3481
|
-
|
|
3482
|
-
if (headerFields && !Array.isArray(headerFields)) {
|
|
3483
|
-
headerFields = [headerFields];
|
|
3484
|
-
}
|
|
3485
|
-
const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
|
|
3658
|
+
const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
|
|
3486
3659
|
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
3487
3660
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3488
3661
|
);
|
|
@@ -3514,20 +3687,8 @@ var init_QvdFileReader = __esm({
|
|
|
3514
3687
|
}
|
|
3515
3688
|
const rowsToLoad = windowRows;
|
|
3516
3689
|
validateSymbolTableSizeEarly(symbolTableLength, this._path);
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
["RecordByteSize", recordSize],
|
|
3520
|
-
["NoOfRecords", totalRows]
|
|
3521
|
-
]) {
|
|
3522
|
-
if (!Number.isSafeInteger(value) || Number(value) < 0) {
|
|
3523
|
-
throw new QvdCorruptedError(`Invalid header value: ${name}`, {
|
|
3524
|
-
name,
|
|
3525
|
-
value,
|
|
3526
|
-
file: this._path,
|
|
3527
|
-
stage: "readData"
|
|
3528
|
-
});
|
|
3529
|
-
}
|
|
3530
|
-
}
|
|
3690
|
+
validateRecordSize(recordSize, this._path, "readData");
|
|
3691
|
+
validateRecordCount(totalRows, this._path, "readData");
|
|
3531
3692
|
const skippedIndexBytes = resolved.offset * recordSize;
|
|
3532
3693
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
3533
3694
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
@@ -3625,36 +3786,12 @@ var init_QvdFileReader = __esm({
|
|
|
3625
3786
|
}
|
|
3626
3787
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3627
3788
|
const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
|
|
3628
|
-
this.
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
file: this._path,
|
|
3632
|
-
stage: "parseHeader"
|
|
3633
|
-
});
|
|
3634
|
-
}
|
|
3635
|
-
validateHeaderStructure(this._header, this._path, "parseHeader");
|
|
3636
|
-
const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
|
|
3637
|
-
const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
|
|
3638
|
-
if (fieldList.length === 0) {
|
|
3639
|
-
throw new QvdCorruptedError("The QVD file header declares no fields", {
|
|
3640
|
-
file: this._path,
|
|
3641
|
-
stage: "parseHeader"
|
|
3642
|
-
});
|
|
3643
|
-
}
|
|
3644
|
-
const malformedIndex = fieldList.findIndex(
|
|
3645
|
-
(field) => field === null || typeof field !== "object" || Array.isArray(field)
|
|
3646
|
-
);
|
|
3647
|
-
if (malformedIndex !== -1) {
|
|
3648
|
-
throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
|
|
3649
|
-
fieldIndex: malformedIndex,
|
|
3650
|
-
fieldCount: fieldList.length,
|
|
3651
|
-
file: this._path,
|
|
3652
|
-
stage: "parseHeader"
|
|
3653
|
-
});
|
|
3654
|
-
}
|
|
3789
|
+
this._fieldBitMetadataValidated = false;
|
|
3790
|
+
this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
|
|
3791
|
+
const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
|
|
3655
3792
|
this._headerOffset = headerBeginIndex;
|
|
3656
3793
|
this._symbolTableOffset = headerEndIndex;
|
|
3657
|
-
this._indexTableOffset = this._symbolTableOffset +
|
|
3794
|
+
this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
|
|
3658
3795
|
this._allFields = fieldList;
|
|
3659
3796
|
this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
|
|
3660
3797
|
}
|
|
@@ -3670,8 +3807,8 @@ var init_QvdFileReader = __esm({
|
|
|
3670
3807
|
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
3671
3808
|
* @param {string} stage Stage name for any error raised here.
|
|
3672
3809
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
3673
|
-
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the
|
|
3674
|
-
* record, so the decoder always counts from zero.
|
|
3810
|
+
* indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
|
|
3811
|
+
* window's first record, file row `firstRow`, so the decoder always counts from zero.
|
|
3675
3812
|
* @private
|
|
3676
3813
|
*/
|
|
3677
3814
|
_planIndexTable(window, stage) {
|
|
@@ -3686,9 +3823,9 @@ var init_QvdFileReader = __esm({
|
|
|
3686
3823
|
}
|
|
3687
3824
|
const allFields = this._allFields;
|
|
3688
3825
|
const fields = this._selectedFields;
|
|
3689
|
-
const recordSize =
|
|
3690
|
-
const totalRows =
|
|
3691
|
-
const indexTableLength =
|
|
3826
|
+
const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
|
|
3827
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3828
|
+
const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
|
|
3692
3829
|
const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
|
|
3693
3830
|
validateIndexTableMetadata(
|
|
3694
3831
|
recordSize,
|
|
@@ -3711,13 +3848,14 @@ var init_QvdFileReader = __esm({
|
|
|
3711
3848
|
for (const field of allFields) {
|
|
3712
3849
|
validateFieldBitMetadata(field, recordSize, this._path);
|
|
3713
3850
|
}
|
|
3851
|
+
validateBitFields(allFields, this._path);
|
|
3714
3852
|
this._fieldBitMetadataValidated = true;
|
|
3715
3853
|
}
|
|
3716
3854
|
assert3(
|
|
3717
3855
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
3718
3856
|
`The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
|
|
3719
3857
|
);
|
|
3720
|
-
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
|
|
3858
|
+
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
|
|
3721
3859
|
}
|
|
3722
3860
|
/**
|
|
3723
3861
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
@@ -3741,9 +3879,12 @@ var init_QvdFileReader = __esm({
|
|
|
3741
3879
|
this._throwIfAborted();
|
|
3742
3880
|
const needed = /* @__PURE__ */ new Set();
|
|
3743
3881
|
symbolUsage[position] = needed;
|
|
3744
|
-
const bitOffset =
|
|
3745
|
-
const bitWidth =
|
|
3746
|
-
const bias =
|
|
3882
|
+
const bitOffset = headerInteger(field["BitOffset"]);
|
|
3883
|
+
const bitWidth = headerInteger(field["BitWidth"]);
|
|
3884
|
+
const bias = headerInteger(field["Bias"]);
|
|
3885
|
+
const length = headerInteger(field["Length"]);
|
|
3886
|
+
let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
|
|
3887
|
+
let counted = false;
|
|
3747
3888
|
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3748
3889
|
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3749
3890
|
decodeIndexColumn(
|
|
@@ -3756,15 +3897,53 @@ var init_QvdFileReader = __esm({
|
|
|
3756
3897
|
column
|
|
3757
3898
|
);
|
|
3758
3899
|
for (let row = 0; row < count; row++) {
|
|
3759
|
-
if (column[row] >= 0) {
|
|
3900
|
+
if (column[row] >= 0 && column[row] < indexLimit) {
|
|
3760
3901
|
needed.add(column[row]);
|
|
3761
3902
|
}
|
|
3762
3903
|
}
|
|
3904
|
+
if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
|
|
3905
|
+
counted = true;
|
|
3906
|
+
indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
|
|
3907
|
+
for (const index of needed) {
|
|
3908
|
+
if (index >= indexLimit) {
|
|
3909
|
+
needed.delete(index);
|
|
3910
|
+
}
|
|
3911
|
+
}
|
|
3912
|
+
}
|
|
3763
3913
|
}
|
|
3764
3914
|
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
3765
3915
|
});
|
|
3766
3916
|
return symbolUsage;
|
|
3767
3917
|
}
|
|
3918
|
+
/**
|
|
3919
|
+
* How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
|
|
3920
|
+
*
|
|
3921
|
+
* The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
|
|
3922
|
+
* `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
|
|
3923
|
+
* validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
|
|
3924
|
+
* same way wherever it is met.
|
|
3925
|
+
*
|
|
3926
|
+
* @param {any} field The field's header.
|
|
3927
|
+
* @return {number} Its symbols.
|
|
3928
|
+
* @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
|
|
3929
|
+
* @private
|
|
3930
|
+
*/
|
|
3931
|
+
_countFieldSymbols(field) {
|
|
3932
|
+
assert3(
|
|
3933
|
+
this._buffer && this._symbolTableOffset && this._indexTableOffset,
|
|
3934
|
+
"The QVD file has not been read before its symbols were counted."
|
|
3935
|
+
);
|
|
3936
|
+
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3937
|
+
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3938
|
+
const offset = headerInteger(field["Offset"]);
|
|
3939
|
+
return countFieldSymbols(
|
|
3940
|
+
symbolBuffer,
|
|
3941
|
+
offset,
|
|
3942
|
+
offset + headerInteger(field["Length"]),
|
|
3943
|
+
field["FieldName"],
|
|
3944
|
+
this._path
|
|
3945
|
+
);
|
|
3946
|
+
}
|
|
3768
3947
|
/**
|
|
3769
3948
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
3770
3949
|
* and should not be called directly.
|
|
@@ -3795,7 +3974,7 @@ var init_QvdFileReader = __esm({
|
|
|
3795
3974
|
const fields = this._selectedFields;
|
|
3796
3975
|
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3797
3976
|
const symbolTableSize = symbolBuffer.length;
|
|
3798
|
-
const totalRows =
|
|
3977
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3799
3978
|
validateSymbolTableSize(symbolTableSize, this._path, totalRows);
|
|
3800
3979
|
if (this._headerMatchesFile) {
|
|
3801
3980
|
validateMemoryAvailability(
|
|
@@ -3813,10 +3992,11 @@ var init_QvdFileReader = __esm({
|
|
|
3813
3992
|
for (const field of allFields) {
|
|
3814
3993
|
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3815
3994
|
}
|
|
3995
|
+
validateSymbolAreas(allFields, this._path);
|
|
3816
3996
|
this._symbolTable = fields.map((field, position) => {
|
|
3817
3997
|
this._throwIfAborted();
|
|
3818
|
-
const symbolsOffset =
|
|
3819
|
-
const symbolsLength =
|
|
3998
|
+
const symbolsOffset = headerInteger(field["Offset"]);
|
|
3999
|
+
const symbolsLength = headerInteger(field["Length"]);
|
|
3820
4000
|
const symbols = parseFieldSymbols(
|
|
3821
4001
|
symbolBuffer,
|
|
3822
4002
|
symbolsOffset,
|
|
@@ -3850,25 +4030,35 @@ var init_QvdFileReader = __esm({
|
|
|
3850
4030
|
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3851
4031
|
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3852
4032
|
*
|
|
4033
|
+
* Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
|
|
4034
|
+
* NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
|
|
4035
|
+
* are built, because every read decodes through this method and a columnar read hands its codes
|
|
4036
|
+
* straight to the caller. Rows outside the window are not decoded, so they are not checked.
|
|
4037
|
+
*
|
|
3853
4038
|
* @param {QvdRowWindow} window The rows to decode.
|
|
4039
|
+
* @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
|
|
3854
4040
|
*/
|
|
3855
4041
|
async _parseIndexTable(window) {
|
|
3856
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
3857
|
-
this.
|
|
3858
|
-
|
|
4042
|
+
const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
|
|
4043
|
+
assert3(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
4044
|
+
const symbolTable = this._symbolTable;
|
|
4045
|
+
const columns = fields.map((field, position) => {
|
|
3859
4046
|
this._throwIfAborted();
|
|
3860
4047
|
const column = decodeIndexColumn(
|
|
3861
4048
|
indexBuffer,
|
|
3862
4049
|
recordSize,
|
|
3863
4050
|
rowsToLoad,
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
new Int32Array(rowsToLoad)
|
|
4051
|
+
headerInteger(field["BitOffset"]),
|
|
4052
|
+
headerInteger(field["BitWidth"]),
|
|
4053
|
+
headerInteger(field["Bias"]),
|
|
4054
|
+
new Int32Array(rowsToLoad),
|
|
4055
|
+
{ symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
|
|
3868
4056
|
);
|
|
3869
4057
|
this._emitProgress("index-table", position + 1, fields.length);
|
|
3870
4058
|
return column;
|
|
3871
4059
|
});
|
|
4060
|
+
this._indexColumns = columns;
|
|
4061
|
+
this._rowsDecoded = rowsToLoad;
|
|
3872
4062
|
}
|
|
3873
4063
|
/**
|
|
3874
4064
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -3890,23 +4080,13 @@ var init_QvdFileReader = __esm({
|
|
|
3890
4080
|
await this._parseHeader();
|
|
3891
4081
|
this._emitProgress("header", 1, 1);
|
|
3892
4082
|
this._throwIfAborted();
|
|
3893
|
-
assert3(this._header, "The QVD file header has not been parsed.");
|
|
4083
|
+
assert3(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
3894
4084
|
const header = this._header["QvdTableHeader"];
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
fields = [fields];
|
|
3898
|
-
}
|
|
3899
|
-
if (fields.length === 0) {
|
|
3900
|
-
throw new QvdCorruptedError("The QVD file header declares no fields", {
|
|
3901
|
-
file: this._path,
|
|
3902
|
-
stage: "readMetadata"
|
|
3903
|
-
});
|
|
3904
|
-
}
|
|
3905
|
-
const columns = fields.map((field) => field["FieldName"]);
|
|
3906
|
-
const rowCount = parseInt(header["NoOfRecords"], 10);
|
|
4085
|
+
const columns = this._allFields.map((field) => field["FieldName"]);
|
|
4086
|
+
const rowCount = headerInteger(header["NoOfRecords"]);
|
|
3907
4087
|
validateRecordCount(rowCount, this._path, "readMetadata");
|
|
3908
4088
|
const shape = new QvdDataFrame([], columns, header, {
|
|
3909
|
-
symbolTableBytes:
|
|
4089
|
+
symbolTableBytes: headerInteger(header["Offset"]),
|
|
3910
4090
|
totalRows: rowCount,
|
|
3911
4091
|
rowsLoaded: 0,
|
|
3912
4092
|
symbolFiltering: false,
|
|
@@ -3997,7 +4177,8 @@ var init_QvdFileReader = __esm({
|
|
|
3997
4177
|
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
3998
4178
|
*
|
|
3999
4179
|
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
4000
|
-
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
4180
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
|
|
4181
|
+
* is still checked, as every read of a file's rows checks it.
|
|
4001
4182
|
*
|
|
4002
4183
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
4003
4184
|
* The rows to cover, in the same spellings `load()` accepts.
|
|
@@ -4015,6 +4196,10 @@ var init_QvdFileReader = __esm({
|
|
|
4015
4196
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
4016
4197
|
const rows = normaliseWindow(window, this._path);
|
|
4017
4198
|
const prepared = await this._prepare(rows, liveRows);
|
|
4199
|
+
if (prepared.rowsAvailable === 0) {
|
|
4200
|
+
this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
|
|
4201
|
+
return;
|
|
4202
|
+
}
|
|
4018
4203
|
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
4019
4204
|
this._throwIfAborted();
|
|
4020
4205
|
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
@@ -4066,8 +4251,8 @@ var init_QvdFileReader = __esm({
|
|
|
4066
4251
|
this._emitProgress("header", 1, 1);
|
|
4067
4252
|
this._throwIfAborted();
|
|
4068
4253
|
assert3(this._header, "The QVD file header has not been parsed.");
|
|
4069
|
-
const totalRows =
|
|
4070
|
-
const symbolTableLength =
|
|
4254
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
4255
|
+
const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
|
|
4071
4256
|
const resolved = resolveWindow(window, totalRows);
|
|
4072
4257
|
const rowsAvailable = resolved.limit;
|
|
4073
4258
|
let symbolsToKeep = null;
|