qvdjs 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/dist/index.cjs +295 -110
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +294 -109
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -5,7 +5,7 @@ var path2 = require('path');
|
|
|
5
5
|
var assert3 = require('assert');
|
|
6
6
|
var crypto2 = require('crypto');
|
|
7
7
|
var promises = require('timers/promises');
|
|
8
|
-
var
|
|
8
|
+
var xml = require('xml2js');
|
|
9
9
|
var os = require('os');
|
|
10
10
|
var v8 = require('v8');
|
|
11
11
|
var util = require('util');
|
|
@@ -16,7 +16,7 @@ var fs2__default = /*#__PURE__*/_interopDefault(fs2);
|
|
|
16
16
|
var path2__default = /*#__PURE__*/_interopDefault(path2);
|
|
17
17
|
var assert3__default = /*#__PURE__*/_interopDefault(assert3);
|
|
18
18
|
var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
|
|
19
|
-
var
|
|
19
|
+
var xml__default = /*#__PURE__*/_interopDefault(xml);
|
|
20
20
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
21
21
|
var v8__default = /*#__PURE__*/_interopDefault(v8);
|
|
22
22
|
|
|
@@ -1098,8 +1098,12 @@ function fieldGeometry(bitOffset, bitWidth) {
|
|
|
1098
1098
|
byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
|
|
1099
1099
|
};
|
|
1100
1100
|
}
|
|
1101
|
-
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
|
|
1101
|
+
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
|
|
1102
|
+
const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
|
|
1102
1103
|
if (bitWidth === 0) {
|
|
1104
|
+
if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
|
|
1105
|
+
refuse2(bounds, 0, bias, bias);
|
|
1106
|
+
}
|
|
1103
1107
|
out.fill(bias, 0, rowCount);
|
|
1104
1108
|
return out;
|
|
1105
1109
|
}
|
|
@@ -1113,10 +1117,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
|
|
|
1113
1117
|
if (byteCount > 2) acc += buffer[base + 2] * 65536;
|
|
1114
1118
|
if (byteCount > 3) acc += buffer[base + 3] * 16777216;
|
|
1115
1119
|
if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
|
|
1116
|
-
|
|
1120
|
+
const index = Math.floor(acc / divisor) % modulus + bias;
|
|
1121
|
+
if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
|
|
1122
|
+
refuse2(bounds, row, index, bias);
|
|
1123
|
+
}
|
|
1124
|
+
out[row] = index;
|
|
1117
1125
|
}
|
|
1118
1126
|
return out;
|
|
1119
1127
|
}
|
|
1128
|
+
function refuse2(bounds, row, index, bias) {
|
|
1129
|
+
throw new exports.QvdCorruptedError("Symbol index out of range", {
|
|
1130
|
+
field: bounds.field,
|
|
1131
|
+
row: bounds.firstRow + row,
|
|
1132
|
+
symbolIndex: index,
|
|
1133
|
+
symbolCount: bounds.symbolCount,
|
|
1134
|
+
bias,
|
|
1135
|
+
file: bounds.file,
|
|
1136
|
+
stage: "parseIndexTable"
|
|
1137
|
+
});
|
|
1138
|
+
}
|
|
1120
1139
|
function writeBitField(buffer, recordBase, geometry, value) {
|
|
1121
1140
|
const { byteStart, shift, byteCount } = geometry;
|
|
1122
1141
|
if (byteCount === 0) {
|
|
@@ -1133,10 +1152,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
|
|
|
1133
1152
|
var MAX_BIT_WIDTH, POW2;
|
|
1134
1153
|
var init_bitUtils = __esm({
|
|
1135
1154
|
"src/util/bitUtils.js"() {
|
|
1155
|
+
init_QvdErrors();
|
|
1136
1156
|
MAX_BIT_WIDTH = 31;
|
|
1137
1157
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
1138
1158
|
__name(fieldGeometry, "fieldGeometry");
|
|
1139
1159
|
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
1160
|
+
__name(refuse2, "refuse");
|
|
1140
1161
|
__name(writeBitField, "writeBitField");
|
|
1141
1162
|
}
|
|
1142
1163
|
});
|
|
@@ -1781,7 +1802,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1781
1802
|
});
|
|
1782
1803
|
}
|
|
1783
1804
|
}
|
|
1784
|
-
const builder = new
|
|
1805
|
+
const builder = new xml__default.default.Builder({
|
|
1785
1806
|
renderOpts: {
|
|
1786
1807
|
pretty: true,
|
|
1787
1808
|
newline: "\r\n",
|
|
@@ -2343,6 +2364,13 @@ var init_memoryUtils = __esm({
|
|
|
2343
2364
|
});
|
|
2344
2365
|
|
|
2345
2366
|
// src/util/validationUtils.js
|
|
2367
|
+
function headerInteger(value) {
|
|
2368
|
+
if (typeof value !== "string" || !/^\s*-?\d+\s*$/.test(value)) {
|
|
2369
|
+
return NaN;
|
|
2370
|
+
}
|
|
2371
|
+
const number = Number(value);
|
|
2372
|
+
return number === 0 ? 0 : number;
|
|
2373
|
+
}
|
|
2346
2374
|
function validateHeaderStructure(headerObj, filePath, stage) {
|
|
2347
2375
|
const tableHeader = headerObj?.["QvdTableHeader"];
|
|
2348
2376
|
if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
|
|
@@ -2352,7 +2380,7 @@ function validateHeaderStructure(headerObj, filePath, stage) {
|
|
|
2352
2380
|
stage
|
|
2353
2381
|
});
|
|
2354
2382
|
}
|
|
2355
|
-
const symbolTableLength =
|
|
2383
|
+
const symbolTableLength = headerInteger(tableHeader["Offset"]);
|
|
2356
2384
|
if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
|
|
2357
2385
|
throw new exports.QvdCorruptedError("Invalid symbol table offset", {
|
|
2358
2386
|
offset: tableHeader["Offset"],
|
|
@@ -2360,6 +2388,47 @@ function validateHeaderStructure(headerObj, filePath, stage) {
|
|
|
2360
2388
|
stage
|
|
2361
2389
|
});
|
|
2362
2390
|
}
|
|
2391
|
+
const fields = tableHeader["Fields"]?.["QvdFieldHeader"];
|
|
2392
|
+
const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
|
|
2393
|
+
if (fieldList.length === 0) {
|
|
2394
|
+
throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
|
|
2395
|
+
file: filePath,
|
|
2396
|
+
stage
|
|
2397
|
+
});
|
|
2398
|
+
}
|
|
2399
|
+
const malformedIndex = fieldList.findIndex(
|
|
2400
|
+
(field) => field === null || typeof field !== "object" || Array.isArray(field)
|
|
2401
|
+
);
|
|
2402
|
+
if (malformedIndex !== -1) {
|
|
2403
|
+
throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
|
|
2404
|
+
fieldIndex: malformedIndex,
|
|
2405
|
+
fieldCount: fieldList.length,
|
|
2406
|
+
file: filePath,
|
|
2407
|
+
stage
|
|
2408
|
+
});
|
|
2409
|
+
}
|
|
2410
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2411
|
+
fieldList.forEach((field, fieldIndex) => {
|
|
2412
|
+
const name = field["FieldName"];
|
|
2413
|
+
if (typeof name !== "string" || name === "") {
|
|
2414
|
+
throw new exports.QvdCorruptedError("The QVD file header declares a field with no usable name", {
|
|
2415
|
+
fieldIndex,
|
|
2416
|
+
fieldName: name,
|
|
2417
|
+
file: filePath,
|
|
2418
|
+
stage
|
|
2419
|
+
});
|
|
2420
|
+
}
|
|
2421
|
+
if (seen.has(name)) {
|
|
2422
|
+
throw new exports.QvdCorruptedError("The QVD file header declares two fields with the same name", {
|
|
2423
|
+
field: name,
|
|
2424
|
+
fieldIndexes: [seen.get(name), fieldIndex],
|
|
2425
|
+
file: filePath,
|
|
2426
|
+
stage
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
seen.set(name, fieldIndex);
|
|
2430
|
+
});
|
|
2431
|
+
return fieldList;
|
|
2363
2432
|
}
|
|
2364
2433
|
function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
|
|
2365
2434
|
const heapLimit = getHeapLimit();
|
|
@@ -2405,8 +2474,8 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
|
|
|
2405
2474
|
}
|
|
2406
2475
|
}
|
|
2407
2476
|
function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
2408
|
-
const symbolsOffset =
|
|
2409
|
-
const symbolsLength =
|
|
2477
|
+
const symbolsOffset = headerInteger(field["Offset"]);
|
|
2478
|
+
const symbolsLength = headerInteger(field["Length"]);
|
|
2410
2479
|
if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
|
|
2411
2480
|
throw new exports.QvdCorruptedError("Invalid symbol offset", {
|
|
2412
2481
|
field: field["FieldName"],
|
|
@@ -2434,6 +2503,63 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
|
2434
2503
|
});
|
|
2435
2504
|
}
|
|
2436
2505
|
}
|
|
2506
|
+
function firstOverlap(ranges) {
|
|
2507
|
+
const claimed = ranges.filter((range) => range.length > 0).sort((a, b) => a.start - b.start);
|
|
2508
|
+
let furthest = null;
|
|
2509
|
+
for (const range of claimed) {
|
|
2510
|
+
if (furthest !== null && range.start < furthest.start + furthest.length) {
|
|
2511
|
+
return [furthest, range];
|
|
2512
|
+
}
|
|
2513
|
+
if (furthest === null || range.start + range.length > furthest.start + furthest.length) {
|
|
2514
|
+
furthest = range;
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
return null;
|
|
2518
|
+
}
|
|
2519
|
+
function validateSymbolAreas(fields, filePath) {
|
|
2520
|
+
const overlap = firstOverlap(
|
|
2521
|
+
fields.map((field) => ({
|
|
2522
|
+
field: field["FieldName"],
|
|
2523
|
+
start: headerInteger(field["Offset"]),
|
|
2524
|
+
length: headerInteger(field["Length"])
|
|
2525
|
+
}))
|
|
2526
|
+
);
|
|
2527
|
+
if (overlap !== null) {
|
|
2528
|
+
const [first, second] = overlap;
|
|
2529
|
+
throw new exports.QvdCorruptedError("Symbol areas overlap", {
|
|
2530
|
+
field: second.field,
|
|
2531
|
+
offset: second.start,
|
|
2532
|
+
length: second.length,
|
|
2533
|
+
overlaps: first.field,
|
|
2534
|
+
overlapsOffset: first.start,
|
|
2535
|
+
overlapsLength: first.length,
|
|
2536
|
+
file: filePath,
|
|
2537
|
+
stage: "parseSymbolTable"
|
|
2538
|
+
});
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
function validateBitFields(fields, filePath) {
|
|
2542
|
+
const overlap = firstOverlap(
|
|
2543
|
+
fields.map((field) => ({
|
|
2544
|
+
field: field["FieldName"],
|
|
2545
|
+
start: headerInteger(field["BitOffset"]),
|
|
2546
|
+
length: headerInteger(field["BitWidth"])
|
|
2547
|
+
}))
|
|
2548
|
+
);
|
|
2549
|
+
if (overlap !== null) {
|
|
2550
|
+
const [first, second] = overlap;
|
|
2551
|
+
throw new exports.QvdCorruptedError("Bit fields overlap", {
|
|
2552
|
+
field: second.field,
|
|
2553
|
+
bitOffset: second.start,
|
|
2554
|
+
bitWidth: second.length,
|
|
2555
|
+
overlaps: first.field,
|
|
2556
|
+
overlapsBitOffset: first.start,
|
|
2557
|
+
overlapsBitWidth: first.length,
|
|
2558
|
+
file: filePath,
|
|
2559
|
+
stage: "parseIndexTable"
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2437
2563
|
function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
2438
2564
|
if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
|
|
2439
2565
|
throw new exports.QvdCorruptedError("Invalid number of records", {
|
|
@@ -2443,14 +2569,17 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
|
2443
2569
|
});
|
|
2444
2570
|
}
|
|
2445
2571
|
}
|
|
2446
|
-
function
|
|
2572
|
+
function validateRecordSize(recordSize, filePath, stage = "parseIndexTable") {
|
|
2447
2573
|
if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
|
|
2448
2574
|
throw new exports.QvdCorruptedError("Invalid record byte size", {
|
|
2449
2575
|
recordSize,
|
|
2450
2576
|
file: filePath,
|
|
2451
|
-
stage
|
|
2577
|
+
stage
|
|
2452
2578
|
});
|
|
2453
2579
|
}
|
|
2580
|
+
}
|
|
2581
|
+
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
|
|
2582
|
+
validateRecordSize(recordSize, filePath);
|
|
2454
2583
|
validateRecordCount(totalRows, filePath);
|
|
2455
2584
|
if (recordSize === 0 && totalRows > 0) {
|
|
2456
2585
|
throw new exports.QvdCorruptedError("Record byte size cannot be zero when records exist", {
|
|
@@ -2534,8 +2663,8 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
|
|
|
2534
2663
|
}
|
|
2535
2664
|
}
|
|
2536
2665
|
function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
2537
|
-
const bitOffset =
|
|
2538
|
-
const bitWidth =
|
|
2666
|
+
const bitOffset = headerInteger(field["BitOffset"]);
|
|
2667
|
+
const bitWidth = headerInteger(field["BitWidth"]);
|
|
2539
2668
|
if (isNaN(bitOffset) || !Number.isSafeInteger(bitOffset) || bitOffset < 0) {
|
|
2540
2669
|
throw new exports.QvdCorruptedError("Invalid bit offset", {
|
|
2541
2670
|
field: field["FieldName"],
|
|
@@ -2552,7 +2681,7 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2552
2681
|
stage: "parseIndexTable"
|
|
2553
2682
|
});
|
|
2554
2683
|
}
|
|
2555
|
-
const bias =
|
|
2684
|
+
const bias = headerInteger(field["Bias"]);
|
|
2556
2685
|
if (isNaN(bias) || !Number.isSafeInteger(bias)) {
|
|
2557
2686
|
throw new exports.QvdCorruptedError("Invalid bias", {
|
|
2558
2687
|
field: field["FieldName"],
|
|
@@ -2570,6 +2699,23 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2570
2699
|
stage: "parseIndexTable"
|
|
2571
2700
|
});
|
|
2572
2701
|
}
|
|
2702
|
+
if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
|
|
2703
|
+
throw new exports.QvdCorruptedError("Bias out of range", {
|
|
2704
|
+
field: field["FieldName"],
|
|
2705
|
+
bias,
|
|
2706
|
+
bitWidth,
|
|
2707
|
+
file: filePath,
|
|
2708
|
+
stage: "parseIndexTable"
|
|
2709
|
+
});
|
|
2710
|
+
}
|
|
2711
|
+
if (bias !== 0 && bias !== -2) {
|
|
2712
|
+
throw new exports.QvdCorruptedError("Bias is neither 0 nor -2", {
|
|
2713
|
+
field: field["FieldName"],
|
|
2714
|
+
bias,
|
|
2715
|
+
file: filePath,
|
|
2716
|
+
stage: "parseIndexTable"
|
|
2717
|
+
});
|
|
2718
|
+
}
|
|
2573
2719
|
const recordSizeInBits = recordSize * 8;
|
|
2574
2720
|
if (bitOffset + bitWidth > recordSizeInBits) {
|
|
2575
2721
|
throw new exports.QvdCorruptedError("Bit field extends beyond record size", {
|
|
@@ -2587,11 +2733,16 @@ var init_validationUtils = __esm({
|
|
|
2587
2733
|
init_QvdErrors();
|
|
2588
2734
|
init_memoryUtils();
|
|
2589
2735
|
init_bitUtils();
|
|
2736
|
+
__name(headerInteger, "headerInteger");
|
|
2590
2737
|
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2591
2738
|
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2592
2739
|
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2593
2740
|
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2741
|
+
__name(firstOverlap, "firstOverlap");
|
|
2742
|
+
__name(validateSymbolAreas, "validateSymbolAreas");
|
|
2743
|
+
__name(validateBitFields, "validateBitFields");
|
|
2594
2744
|
__name(validateRecordCount, "validateRecordCount");
|
|
2745
|
+
__name(validateRecordSize, "validateRecordSize");
|
|
2595
2746
|
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2596
2747
|
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
2597
2748
|
}
|
|
@@ -2697,7 +2848,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2697
2848
|
}
|
|
2698
2849
|
return { numbers, texts };
|
|
2699
2850
|
}
|
|
2700
|
-
|
|
2851
|
+
function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
|
|
2852
|
+
return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
|
|
2853
|
+
}
|
|
2854
|
+
var MAX_TEXT_BYTES, DECODE_NOTHING;
|
|
2701
2855
|
var init_symbolParser = __esm({
|
|
2702
2856
|
"src/util/symbolParser.js"() {
|
|
2703
2857
|
init_QvdErrors();
|
|
@@ -2705,6 +2859,8 @@ var init_symbolParser = __esm({
|
|
|
2705
2859
|
__name(textEnd, "textEnd");
|
|
2706
2860
|
__name(overflow, "overflow");
|
|
2707
2861
|
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2862
|
+
DECODE_NOTHING = /* @__PURE__ */ new Set();
|
|
2863
|
+
__name(countFieldSymbols, "countFieldSymbols");
|
|
2708
2864
|
}
|
|
2709
2865
|
});
|
|
2710
2866
|
|
|
@@ -3218,7 +3374,32 @@ async function* chunksFrom(handle, chunkSize, failed) {
|
|
|
3218
3374
|
position += bytesRead;
|
|
3219
3375
|
}
|
|
3220
3376
|
}
|
|
3221
|
-
|
|
3377
|
+
async function parseHeaderXml(text, file, stage) {
|
|
3378
|
+
let parsed;
|
|
3379
|
+
try {
|
|
3380
|
+
parsed = await xml__default.default.parseStringPromise(text, { explicitArray: false });
|
|
3381
|
+
} catch (error) {
|
|
3382
|
+
throw new exports.QvdParseError(
|
|
3383
|
+
"The XML header could not be parsed.",
|
|
3384
|
+
{
|
|
3385
|
+
// The first line of the parser's message. The rest is the line and column it stopped at, which
|
|
3386
|
+
// stay on `cause`.
|
|
3387
|
+
reason: String(
|
|
3388
|
+
/** @type {any} */
|
|
3389
|
+
error?.message ?? error
|
|
3390
|
+
).split("\n")[0],
|
|
3391
|
+
file,
|
|
3392
|
+
stage
|
|
3393
|
+
},
|
|
3394
|
+
{ cause: error }
|
|
3395
|
+
);
|
|
3396
|
+
}
|
|
3397
|
+
if (!parsed) {
|
|
3398
|
+
throw new exports.QvdParseError("The XML header could not be parsed.", { file, stage });
|
|
3399
|
+
}
|
|
3400
|
+
return parsed;
|
|
3401
|
+
}
|
|
3402
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
|
|
3222
3403
|
var init_QvdFileReader = __esm({
|
|
3223
3404
|
"src/QvdFileReader.js"() {
|
|
3224
3405
|
init_QvdDataFrame();
|
|
@@ -3236,7 +3417,9 @@ var init_QvdFileReader = __esm({
|
|
|
3236
3417
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
3237
3418
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
3238
3419
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
3420
|
+
COUNT_SYMBOLS_PAST = 65536;
|
|
3239
3421
|
__name(chunksFrom, "chunksFrom");
|
|
3422
|
+
__name(parseHeaderXml, "parseHeaderXml");
|
|
3240
3423
|
exports.QvdFileReader = class {
|
|
3241
3424
|
static {
|
|
3242
3425
|
__name(this, "QvdFileReader");
|
|
@@ -3472,29 +3655,19 @@ var init_QvdFileReader = __esm({
|
|
|
3472
3655
|
const headerBuffer = Buffer.concat(headerChunks);
|
|
3473
3656
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3474
3657
|
const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
|
|
3475
|
-
const headerObj = await
|
|
3476
|
-
|
|
3477
|
-
throw new exports.QvdParseError("The XML header could not be parsed.", {
|
|
3478
|
-
file: this._path,
|
|
3479
|
-
stage: "readData"
|
|
3480
|
-
});
|
|
3481
|
-
}
|
|
3482
|
-
validateHeaderStructure(headerObj, this._path, "readData");
|
|
3658
|
+
const headerObj = await parseHeaderXml(headerXml, this._path, "readData");
|
|
3659
|
+
const headerFields = validateHeaderStructure(headerObj, this._path, "readData");
|
|
3483
3660
|
const symbolTableOffset = headerEndIndex;
|
|
3484
|
-
const symbolTableLength =
|
|
3661
|
+
const symbolTableLength = headerInteger(headerObj["QvdTableHeader"]["Offset"]);
|
|
3485
3662
|
const indexTableOffset = symbolTableOffset + symbolTableLength;
|
|
3486
|
-
const recordSize =
|
|
3487
|
-
const totalRows =
|
|
3663
|
+
const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
|
|
3664
|
+
const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
|
|
3488
3665
|
if (headerOnly) {
|
|
3489
3666
|
this._buffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3490
3667
|
this._emitProgress("read", 1, 1);
|
|
3491
3668
|
return;
|
|
3492
3669
|
}
|
|
3493
|
-
|
|
3494
|
-
if (headerFields && !Array.isArray(headerFields)) {
|
|
3495
|
-
headerFields = [headerFields];
|
|
3496
|
-
}
|
|
3497
|
-
const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
|
|
3670
|
+
const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
|
|
3498
3671
|
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
3499
3672
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3500
3673
|
);
|
|
@@ -3526,20 +3699,8 @@ var init_QvdFileReader = __esm({
|
|
|
3526
3699
|
}
|
|
3527
3700
|
const rowsToLoad = windowRows;
|
|
3528
3701
|
validateSymbolTableSizeEarly(symbolTableLength, this._path);
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
["RecordByteSize", recordSize],
|
|
3532
|
-
["NoOfRecords", totalRows]
|
|
3533
|
-
]) {
|
|
3534
|
-
if (!Number.isSafeInteger(value) || Number(value) < 0) {
|
|
3535
|
-
throw new exports.QvdCorruptedError(`Invalid header value: ${name}`, {
|
|
3536
|
-
name,
|
|
3537
|
-
value,
|
|
3538
|
-
file: this._path,
|
|
3539
|
-
stage: "readData"
|
|
3540
|
-
});
|
|
3541
|
-
}
|
|
3542
|
-
}
|
|
3702
|
+
validateRecordSize(recordSize, this._path, "readData");
|
|
3703
|
+
validateRecordCount(totalRows, this._path, "readData");
|
|
3543
3704
|
const skippedIndexBytes = resolved.offset * recordSize;
|
|
3544
3705
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
3545
3706
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
@@ -3637,36 +3798,12 @@ var init_QvdFileReader = __esm({
|
|
|
3637
3798
|
}
|
|
3638
3799
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3639
3800
|
const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
|
|
3640
|
-
this.
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
file: this._path,
|
|
3644
|
-
stage: "parseHeader"
|
|
3645
|
-
});
|
|
3646
|
-
}
|
|
3647
|
-
validateHeaderStructure(this._header, this._path, "parseHeader");
|
|
3648
|
-
const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
|
|
3649
|
-
const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
|
|
3650
|
-
if (fieldList.length === 0) {
|
|
3651
|
-
throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
|
|
3652
|
-
file: this._path,
|
|
3653
|
-
stage: "parseHeader"
|
|
3654
|
-
});
|
|
3655
|
-
}
|
|
3656
|
-
const malformedIndex = fieldList.findIndex(
|
|
3657
|
-
(field) => field === null || typeof field !== "object" || Array.isArray(field)
|
|
3658
|
-
);
|
|
3659
|
-
if (malformedIndex !== -1) {
|
|
3660
|
-
throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
|
|
3661
|
-
fieldIndex: malformedIndex,
|
|
3662
|
-
fieldCount: fieldList.length,
|
|
3663
|
-
file: this._path,
|
|
3664
|
-
stage: "parseHeader"
|
|
3665
|
-
});
|
|
3666
|
-
}
|
|
3801
|
+
this._fieldBitMetadataValidated = false;
|
|
3802
|
+
this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
|
|
3803
|
+
const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
|
|
3667
3804
|
this._headerOffset = headerBeginIndex;
|
|
3668
3805
|
this._symbolTableOffset = headerEndIndex;
|
|
3669
|
-
this._indexTableOffset = this._symbolTableOffset +
|
|
3806
|
+
this._indexTableOffset = this._symbolTableOffset + headerInteger(this._header["QvdTableHeader"]["Offset"]);
|
|
3670
3807
|
this._allFields = fieldList;
|
|
3671
3808
|
this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
|
|
3672
3809
|
}
|
|
@@ -3682,8 +3819,8 @@ var init_QvdFileReader = __esm({
|
|
|
3682
3819
|
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
3683
3820
|
* @param {string} stage Stage name for any error raised here.
|
|
3684
3821
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
3685
|
-
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the
|
|
3686
|
-
* record, so the decoder always counts from zero.
|
|
3822
|
+
* indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
|
|
3823
|
+
* window's first record, file row `firstRow`, so the decoder always counts from zero.
|
|
3687
3824
|
* @private
|
|
3688
3825
|
*/
|
|
3689
3826
|
_planIndexTable(window, stage) {
|
|
@@ -3698,9 +3835,9 @@ var init_QvdFileReader = __esm({
|
|
|
3698
3835
|
}
|
|
3699
3836
|
const allFields = this._allFields;
|
|
3700
3837
|
const fields = this._selectedFields;
|
|
3701
|
-
const recordSize =
|
|
3702
|
-
const totalRows =
|
|
3703
|
-
const indexTableLength =
|
|
3838
|
+
const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
|
|
3839
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3840
|
+
const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
|
|
3704
3841
|
const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
|
|
3705
3842
|
validateIndexTableMetadata(
|
|
3706
3843
|
recordSize,
|
|
@@ -3723,13 +3860,14 @@ var init_QvdFileReader = __esm({
|
|
|
3723
3860
|
for (const field of allFields) {
|
|
3724
3861
|
validateFieldBitMetadata(field, recordSize, this._path);
|
|
3725
3862
|
}
|
|
3863
|
+
validateBitFields(allFields, this._path);
|
|
3726
3864
|
this._fieldBitMetadataValidated = true;
|
|
3727
3865
|
}
|
|
3728
3866
|
assert3__default.default(
|
|
3729
3867
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
3730
3868
|
`The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
|
|
3731
3869
|
);
|
|
3732
|
-
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
|
|
3870
|
+
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
|
|
3733
3871
|
}
|
|
3734
3872
|
/**
|
|
3735
3873
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
@@ -3753,9 +3891,12 @@ var init_QvdFileReader = __esm({
|
|
|
3753
3891
|
this._throwIfAborted();
|
|
3754
3892
|
const needed = /* @__PURE__ */ new Set();
|
|
3755
3893
|
symbolUsage[position] = needed;
|
|
3756
|
-
const bitOffset =
|
|
3757
|
-
const bitWidth =
|
|
3758
|
-
const bias =
|
|
3894
|
+
const bitOffset = headerInteger(field["BitOffset"]);
|
|
3895
|
+
const bitWidth = headerInteger(field["BitWidth"]);
|
|
3896
|
+
const bias = headerInteger(field["Bias"]);
|
|
3897
|
+
const length = headerInteger(field["Length"]);
|
|
3898
|
+
let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
|
|
3899
|
+
let counted = false;
|
|
3759
3900
|
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3760
3901
|
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3761
3902
|
decodeIndexColumn(
|
|
@@ -3768,15 +3909,53 @@ var init_QvdFileReader = __esm({
|
|
|
3768
3909
|
column
|
|
3769
3910
|
);
|
|
3770
3911
|
for (let row = 0; row < count; row++) {
|
|
3771
|
-
if (column[row] >= 0) {
|
|
3912
|
+
if (column[row] >= 0 && column[row] < indexLimit) {
|
|
3772
3913
|
needed.add(column[row]);
|
|
3773
3914
|
}
|
|
3774
3915
|
}
|
|
3916
|
+
if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
|
|
3917
|
+
counted = true;
|
|
3918
|
+
indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
|
|
3919
|
+
for (const index of needed) {
|
|
3920
|
+
if (index >= indexLimit) {
|
|
3921
|
+
needed.delete(index);
|
|
3922
|
+
}
|
|
3923
|
+
}
|
|
3924
|
+
}
|
|
3775
3925
|
}
|
|
3776
3926
|
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
3777
3927
|
});
|
|
3778
3928
|
return symbolUsage;
|
|
3779
3929
|
}
|
|
3930
|
+
/**
|
|
3931
|
+
* How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
|
|
3932
|
+
*
|
|
3933
|
+
* The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
|
|
3934
|
+
* `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
|
|
3935
|
+
* validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
|
|
3936
|
+
* same way wherever it is met.
|
|
3937
|
+
*
|
|
3938
|
+
* @param {any} field The field's header.
|
|
3939
|
+
* @return {number} Its symbols.
|
|
3940
|
+
* @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
|
|
3941
|
+
* @private
|
|
3942
|
+
*/
|
|
3943
|
+
_countFieldSymbols(field) {
|
|
3944
|
+
assert3__default.default(
|
|
3945
|
+
this._buffer && this._symbolTableOffset && this._indexTableOffset,
|
|
3946
|
+
"The QVD file has not been read before its symbols were counted."
|
|
3947
|
+
);
|
|
3948
|
+
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3949
|
+
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3950
|
+
const offset = headerInteger(field["Offset"]);
|
|
3951
|
+
return countFieldSymbols(
|
|
3952
|
+
symbolBuffer,
|
|
3953
|
+
offset,
|
|
3954
|
+
offset + headerInteger(field["Length"]),
|
|
3955
|
+
field["FieldName"],
|
|
3956
|
+
this._path
|
|
3957
|
+
);
|
|
3958
|
+
}
|
|
3780
3959
|
/**
|
|
3781
3960
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
3782
3961
|
* and should not be called directly.
|
|
@@ -3807,7 +3986,7 @@ var init_QvdFileReader = __esm({
|
|
|
3807
3986
|
const fields = this._selectedFields;
|
|
3808
3987
|
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3809
3988
|
const symbolTableSize = symbolBuffer.length;
|
|
3810
|
-
const totalRows =
|
|
3989
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3811
3990
|
validateSymbolTableSize(symbolTableSize, this._path, totalRows);
|
|
3812
3991
|
if (this._headerMatchesFile) {
|
|
3813
3992
|
validateMemoryAvailability(
|
|
@@ -3825,10 +4004,11 @@ var init_QvdFileReader = __esm({
|
|
|
3825
4004
|
for (const field of allFields) {
|
|
3826
4005
|
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3827
4006
|
}
|
|
4007
|
+
validateSymbolAreas(allFields, this._path);
|
|
3828
4008
|
this._symbolTable = fields.map((field, position) => {
|
|
3829
4009
|
this._throwIfAborted();
|
|
3830
|
-
const symbolsOffset =
|
|
3831
|
-
const symbolsLength =
|
|
4010
|
+
const symbolsOffset = headerInteger(field["Offset"]);
|
|
4011
|
+
const symbolsLength = headerInteger(field["Length"]);
|
|
3832
4012
|
const symbols = parseFieldSymbols(
|
|
3833
4013
|
symbolBuffer,
|
|
3834
4014
|
symbolsOffset,
|
|
@@ -3862,25 +4042,35 @@ var init_QvdFileReader = __esm({
|
|
|
3862
4042
|
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3863
4043
|
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3864
4044
|
*
|
|
4045
|
+
* Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
|
|
4046
|
+
* NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
|
|
4047
|
+
* are built, because every read decodes through this method and a columnar read hands its codes
|
|
4048
|
+
* straight to the caller. Rows outside the window are not decoded, so they are not checked.
|
|
4049
|
+
*
|
|
3865
4050
|
* @param {QvdRowWindow} window The rows to decode.
|
|
4051
|
+
* @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
|
|
3866
4052
|
*/
|
|
3867
4053
|
async _parseIndexTable(window) {
|
|
3868
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
3869
|
-
this.
|
|
3870
|
-
|
|
4054
|
+
const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
|
|
4055
|
+
assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
4056
|
+
const symbolTable = this._symbolTable;
|
|
4057
|
+
const columns = fields.map((field, position) => {
|
|
3871
4058
|
this._throwIfAborted();
|
|
3872
4059
|
const column = decodeIndexColumn(
|
|
3873
4060
|
indexBuffer,
|
|
3874
4061
|
recordSize,
|
|
3875
4062
|
rowsToLoad,
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
new Int32Array(rowsToLoad)
|
|
4063
|
+
headerInteger(field["BitOffset"]),
|
|
4064
|
+
headerInteger(field["BitWidth"]),
|
|
4065
|
+
headerInteger(field["Bias"]),
|
|
4066
|
+
new Int32Array(rowsToLoad),
|
|
4067
|
+
{ symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
|
|
3880
4068
|
);
|
|
3881
4069
|
this._emitProgress("index-table", position + 1, fields.length);
|
|
3882
4070
|
return column;
|
|
3883
4071
|
});
|
|
4072
|
+
this._indexColumns = columns;
|
|
4073
|
+
this._rowsDecoded = rowsToLoad;
|
|
3884
4074
|
}
|
|
3885
4075
|
/**
|
|
3886
4076
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -3902,23 +4092,13 @@ var init_QvdFileReader = __esm({
|
|
|
3902
4092
|
await this._parseHeader();
|
|
3903
4093
|
this._emitProgress("header", 1, 1);
|
|
3904
4094
|
this._throwIfAborted();
|
|
3905
|
-
assert3__default.default(this._header, "The QVD file header has not been parsed.");
|
|
4095
|
+
assert3__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
3906
4096
|
const header = this._header["QvdTableHeader"];
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
fields = [fields];
|
|
3910
|
-
}
|
|
3911
|
-
if (fields.length === 0) {
|
|
3912
|
-
throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
|
|
3913
|
-
file: this._path,
|
|
3914
|
-
stage: "readMetadata"
|
|
3915
|
-
});
|
|
3916
|
-
}
|
|
3917
|
-
const columns = fields.map((field) => field["FieldName"]);
|
|
3918
|
-
const rowCount = parseInt(header["NoOfRecords"], 10);
|
|
4097
|
+
const columns = this._allFields.map((field) => field["FieldName"]);
|
|
4098
|
+
const rowCount = headerInteger(header["NoOfRecords"]);
|
|
3919
4099
|
validateRecordCount(rowCount, this._path, "readMetadata");
|
|
3920
4100
|
const shape = new exports.QvdDataFrame([], columns, header, {
|
|
3921
|
-
symbolTableBytes:
|
|
4101
|
+
symbolTableBytes: headerInteger(header["Offset"]),
|
|
3922
4102
|
totalRows: rowCount,
|
|
3923
4103
|
rowsLoaded: 0,
|
|
3924
4104
|
symbolFiltering: false,
|
|
@@ -4009,7 +4189,8 @@ var init_QvdFileReader = __esm({
|
|
|
4009
4189
|
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
4010
4190
|
*
|
|
4011
4191
|
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
4012
|
-
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
4192
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
|
|
4193
|
+
* is still checked, as every read of a file's rows checks it.
|
|
4013
4194
|
*
|
|
4014
4195
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
4015
4196
|
* The rows to cover, in the same spellings `load()` accepts.
|
|
@@ -4027,6 +4208,10 @@ var init_QvdFileReader = __esm({
|
|
|
4027
4208
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
4028
4209
|
const rows = normaliseWindow(window, this._path);
|
|
4029
4210
|
const prepared = await this._prepare(rows, liveRows);
|
|
4211
|
+
if (prepared.rowsAvailable === 0) {
|
|
4212
|
+
this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
|
|
4213
|
+
return;
|
|
4214
|
+
}
|
|
4030
4215
|
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
4031
4216
|
this._throwIfAborted();
|
|
4032
4217
|
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
@@ -4078,8 +4263,8 @@ var init_QvdFileReader = __esm({
|
|
|
4078
4263
|
this._emitProgress("header", 1, 1);
|
|
4079
4264
|
this._throwIfAborted();
|
|
4080
4265
|
assert3__default.default(this._header, "The QVD file header has not been parsed.");
|
|
4081
|
-
const totalRows =
|
|
4082
|
-
const symbolTableLength =
|
|
4266
|
+
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
4267
|
+
const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
|
|
4083
4268
|
const resolved = resolveWindow(window, totalRows);
|
|
4084
4269
|
const rowsAvailable = resolved.limit;
|
|
4085
4270
|
let symbolsToKeep = null;
|