qvdjs 2.0.5 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -6
- package/dist/index.cjs +1134 -363
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1133 -362
- package/dist/index.js.map +1 -1
- package/package.json +2 -1
package/dist/index.cjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
var fs2 = require('fs');
|
|
4
4
|
var path2 = require('path');
|
|
5
|
-
var
|
|
5
|
+
var assert4 = require('assert');
|
|
6
6
|
var crypto2 = require('crypto');
|
|
7
7
|
var promises = require('timers/promises');
|
|
8
8
|
var xml = require('xml2js');
|
|
@@ -14,7 +14,7 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
14
14
|
|
|
15
15
|
var fs2__default = /*#__PURE__*/_interopDefault(fs2);
|
|
16
16
|
var path2__default = /*#__PURE__*/_interopDefault(path2);
|
|
17
|
-
var
|
|
17
|
+
var assert4__default = /*#__PURE__*/_interopDefault(assert4);
|
|
18
18
|
var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
|
|
19
19
|
var xml__default = /*#__PURE__*/_interopDefault(xml);
|
|
20
20
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
@@ -453,7 +453,9 @@ var init_optionTypes = __esm({
|
|
|
453
453
|
function requireRowCount(value, name, filePath) {
|
|
454
454
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
455
455
|
throw new exports.QvdValidationError(`${name} must be a non-negative integer`, {
|
|
456
|
+
reason: "option",
|
|
456
457
|
option: name,
|
|
458
|
+
value,
|
|
457
459
|
provided: value,
|
|
458
460
|
type: typeof value,
|
|
459
461
|
file: filePath
|
|
@@ -470,6 +472,9 @@ function normaliseWindow(window, filePath) {
|
|
|
470
472
|
}
|
|
471
473
|
if (typeof window !== "object" || Array.isArray(window)) {
|
|
472
474
|
throw new exports.QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
|
|
475
|
+
reason: "option",
|
|
476
|
+
option: "limit",
|
|
477
|
+
value: window,
|
|
473
478
|
provided: window,
|
|
474
479
|
type: typeof window,
|
|
475
480
|
file: filePath
|
|
@@ -480,6 +485,9 @@ function normaliseWindow(window, filePath) {
|
|
|
480
485
|
const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
|
|
481
486
|
if (limitGiven && maxRowsGiven) {
|
|
482
487
|
throw new exports.QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
|
|
488
|
+
reason: "option",
|
|
489
|
+
option: "limit",
|
|
490
|
+
value: limit,
|
|
483
491
|
maxRows,
|
|
484
492
|
limit,
|
|
485
493
|
file: filePath
|
|
@@ -490,6 +498,19 @@ function normaliseWindow(window, filePath) {
|
|
|
490
498
|
limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
|
|
491
499
|
};
|
|
492
500
|
}
|
|
501
|
+
function requireChunkSize(chunkSize, filePath) {
|
|
502
|
+
if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
503
|
+
throw new exports.QvdValidationError("chunkSize must be a positive integer", {
|
|
504
|
+
reason: "option",
|
|
505
|
+
option: "chunkSize",
|
|
506
|
+
value: chunkSize,
|
|
507
|
+
provided: chunkSize,
|
|
508
|
+
type: typeof chunkSize,
|
|
509
|
+
file: filePath
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
return chunkSize;
|
|
513
|
+
}
|
|
493
514
|
function resolveWindow(window, totalRows) {
|
|
494
515
|
const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
|
|
495
516
|
const offset = Math.min(window.offset, rows);
|
|
@@ -504,6 +525,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
504
525
|
}
|
|
505
526
|
if (!Array.isArray(requested)) {
|
|
506
527
|
throw new exports.QvdValidationError("fields must be an array of field names", {
|
|
528
|
+
reason: "option",
|
|
529
|
+
option: "fields",
|
|
530
|
+
value: requested,
|
|
507
531
|
provided: requested,
|
|
508
532
|
type: typeof requested,
|
|
509
533
|
file: filePath
|
|
@@ -512,6 +536,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
512
536
|
const available = fields.map((field) => field["FieldName"]);
|
|
513
537
|
if (requested.length === 0) {
|
|
514
538
|
throw new exports.QvdValidationError("fields must name at least one field", {
|
|
539
|
+
reason: "option",
|
|
540
|
+
option: "fields",
|
|
541
|
+
value: requested,
|
|
515
542
|
availableColumns: available,
|
|
516
543
|
file: filePath
|
|
517
544
|
});
|
|
@@ -520,6 +547,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
520
547
|
return requested.map((name) => {
|
|
521
548
|
if (typeof name !== "string") {
|
|
522
549
|
throw new exports.QvdValidationError("Field names must be strings", {
|
|
550
|
+
reason: "option",
|
|
551
|
+
option: "fields",
|
|
552
|
+
value: name,
|
|
523
553
|
provided: name,
|
|
524
554
|
type: typeof name,
|
|
525
555
|
availableColumns: available,
|
|
@@ -528,6 +558,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
528
558
|
}
|
|
529
559
|
if (seen.has(name)) {
|
|
530
560
|
throw new exports.QvdValidationError(`Field '${name}' is listed twice`, {
|
|
561
|
+
reason: "option",
|
|
562
|
+
option: "fields",
|
|
563
|
+
value: name,
|
|
531
564
|
column: name,
|
|
532
565
|
fields: requested,
|
|
533
566
|
file: filePath
|
|
@@ -537,6 +570,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
537
570
|
const index = available.indexOf(name);
|
|
538
571
|
if (index === -1) {
|
|
539
572
|
throw new exports.QvdValidationError(`Column '${name}' does not exist`, {
|
|
573
|
+
reason: "option",
|
|
574
|
+
option: "fields",
|
|
575
|
+
value: name,
|
|
540
576
|
column: name,
|
|
541
577
|
availableColumns: available,
|
|
542
578
|
file: filePath
|
|
@@ -551,7 +587,9 @@ function normaliseDuals(value, filePath) {
|
|
|
551
587
|
}
|
|
552
588
|
if (!DUAL_MODES.includes(value)) {
|
|
553
589
|
throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
590
|
+
reason: "option",
|
|
554
591
|
option: "duals",
|
|
592
|
+
value,
|
|
555
593
|
provided: value,
|
|
556
594
|
file: filePath
|
|
557
595
|
});
|
|
@@ -590,6 +628,7 @@ var init_readOptions = __esm({
|
|
|
590
628
|
init_optionTypes();
|
|
591
629
|
__name(requireRowCount, "requireRowCount");
|
|
592
630
|
__name(normaliseWindow, "normaliseWindow");
|
|
631
|
+
__name(requireChunkSize, "requireChunkSize");
|
|
593
632
|
__name(resolveWindow, "resolveWindow");
|
|
594
633
|
__name(selectFields, "selectFields");
|
|
595
634
|
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
@@ -971,7 +1010,7 @@ function changedAfterCheck(checked, change) {
|
|
|
971
1010
|
});
|
|
972
1011
|
}
|
|
973
1012
|
async function openChecked(checked, purpose, failed, { nofollow = NOFOLLOW } = {}) {
|
|
974
|
-
|
|
1013
|
+
assert4__default.default(purpose === "read" || checked.stats !== null, "A rewrite in place is of a file that exists.");
|
|
975
1014
|
const noFollow = checked.onDisk ? nofollow : 0;
|
|
976
1015
|
const flags = (purpose === "rewrite" ? O_WRONLY : O_RDONLY) | noFollow;
|
|
977
1016
|
let handle;
|
|
@@ -1547,9 +1586,9 @@ var init_QvdFileWriter = __esm({
|
|
|
1547
1586
|
* Writes the data to the QVD file.
|
|
1548
1587
|
*/
|
|
1549
1588
|
async _writeData() {
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1589
|
+
assert4__default.default(this._header, "The QVD file header has not been parsed.");
|
|
1590
|
+
assert4__default.default(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
|
|
1591
|
+
assert4__default.default(this._indexBuffer, "The QVD file index table has not been parsed.");
|
|
1553
1592
|
this._emitProgress("write", 0, 1);
|
|
1554
1593
|
const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
|
|
1555
1594
|
const failed = rethrowAsIoError(this._path, "write");
|
|
@@ -1997,7 +2036,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1997
2036
|
const key = keys[slot];
|
|
1998
2037
|
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1999
2038
|
}
|
|
2000
|
-
|
|
2039
|
+
assert4__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
2001
2040
|
columnBuffers.push(columnBuffer);
|
|
2002
2041
|
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
2003
2042
|
this._symbolCounts?.push(keys.length);
|
|
@@ -2036,9 +2075,9 @@ var init_QvdFileWriter = __esm({
|
|
|
2036
2075
|
* @private
|
|
2037
2076
|
*/
|
|
2038
2077
|
_buildIndexTable() {
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2078
|
+
assert4__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
2079
|
+
assert4__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
2080
|
+
assert4__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
2042
2081
|
this._indexTableMetadata = [];
|
|
2043
2082
|
const columns = this._df.columns;
|
|
2044
2083
|
const data = this._df.data;
|
|
@@ -2218,20 +2257,57 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
|
|
|
2218
2257
|
}
|
|
2219
2258
|
return low;
|
|
2220
2259
|
}
|
|
2221
|
-
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2260
|
+
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null) {
|
|
2261
|
+
const answer = checkMemory({
|
|
2262
|
+
symbolTableSize,
|
|
2263
|
+
maxRows,
|
|
2264
|
+
totalRows,
|
|
2265
|
+
safetyFactor,
|
|
2266
|
+
columnCount,
|
|
2267
|
+
materialisesRows,
|
|
2268
|
+
live,
|
|
2269
|
+
bytesHeld,
|
|
2270
|
+
readBytes
|
|
2271
|
+
});
|
|
2272
|
+
if (answer.fits) {
|
|
2226
2273
|
return;
|
|
2227
2274
|
}
|
|
2228
|
-
const
|
|
2275
|
+
const { message, context } = answer.refusal;
|
|
2276
|
+
throw new exports.QvdValidationError(message, {
|
|
2277
|
+
file: filePath,
|
|
2278
|
+
...context,
|
|
2279
|
+
reason: "memory",
|
|
2280
|
+
check: answerOf(answer)
|
|
2281
|
+
});
|
|
2282
|
+
}
|
|
2283
|
+
function checkMemory({
|
|
2284
|
+
symbolTableSize,
|
|
2285
|
+
maxRows,
|
|
2286
|
+
totalRows,
|
|
2287
|
+
safetyFactor = 0.8,
|
|
2288
|
+
columnCount = 0,
|
|
2289
|
+
materialisesRows = true,
|
|
2290
|
+
live = null,
|
|
2291
|
+
bytesHeld = null,
|
|
2292
|
+
readBytes = null,
|
|
2293
|
+
measured = null
|
|
2294
|
+
}) {
|
|
2295
|
+
if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
|
|
2296
|
+
throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
|
|
2297
|
+
safetyFactor,
|
|
2298
|
+
reason: "option",
|
|
2299
|
+
option: "memorySafetyFactor",
|
|
2300
|
+
value: safetyFactor
|
|
2301
|
+
});
|
|
2302
|
+
}
|
|
2303
|
+
const budget = measured ?? getMemoryBudget();
|
|
2229
2304
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
2230
2305
|
const rowsLive = live === null ? null : live.rows;
|
|
2231
2306
|
const liveRowsPerChunk = live === null ? 1 : live.perChunk;
|
|
2232
2307
|
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
2233
2308
|
const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
|
|
2234
|
-
const
|
|
2309
|
+
const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
|
|
2310
|
+
const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
|
|
2235
2311
|
const bounded = budget.candidates.map((candidate) => {
|
|
2236
2312
|
const heapOnly = candidate.source === "V8 heap limit";
|
|
2237
2313
|
return {
|
|
@@ -2242,25 +2318,45 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2242
2318
|
bounds: heapOnly ? "the V8 heap" : "the whole process"
|
|
2243
2319
|
};
|
|
2244
2320
|
});
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2321
|
+
if (safetyFactor === 0) {
|
|
2322
|
+
const lowest = budget.candidates.reduce((least, candidate) => candidate.bytes < least.bytes ? candidate : least);
|
|
2323
|
+
return {
|
|
2324
|
+
fits: true,
|
|
2325
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2326
|
+
// The ceilings are still there and still named - what is missing is any measurement against them.
|
|
2327
|
+
// An earlier version reported `bound: 'none'` here, a third value in a two-value vocabulary that a
|
|
2328
|
+
// caller switching on the documented two would fall straight through.
|
|
2329
|
+
budget: {
|
|
2330
|
+
...budgetOf(budget, { ...lowest, heapOnly: lowest.source === "V8 heap limit" }, 0),
|
|
2331
|
+
allowedBytes: Infinity
|
|
2332
|
+
},
|
|
2333
|
+
exact: symbolTableSize === 0,
|
|
2334
|
+
suggestions: []
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
const tightest = bounded.reduce(
|
|
2338
|
+
(worst, candidate) => candidate.needs / candidate.allowed > worst.needs / worst.allowed ? candidate : worst
|
|
2249
2339
|
);
|
|
2340
|
+
const binding = tightest.needs > tightest.allowed ? tightest : null;
|
|
2250
2341
|
const heapLimit = getHeapLimit();
|
|
2251
2342
|
const availableMemory = binding ? binding.bytes : budget.bytes;
|
|
2252
2343
|
const estimatedMemory = binding ? binding.needs : heapMemory;
|
|
2253
2344
|
const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
|
|
2254
2345
|
if (binding) {
|
|
2255
2346
|
const includeExternal = !binding.heapOnly;
|
|
2256
|
-
const
|
|
2257
|
-
|
|
2347
|
+
const rowsHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForRows(rows) : 0), "rowsHeldBudget");
|
|
2348
|
+
const fitting = /* @__PURE__ */ __name((rows) => recommendedRowsFor(
|
|
2349
|
+
rowsHeldBudget(rows),
|
|
2258
2350
|
symbolTableSize,
|
|
2259
2351
|
totalRows,
|
|
2260
2352
|
columnCount,
|
|
2261
2353
|
materialisesRows,
|
|
2262
2354
|
includeExternal
|
|
2263
|
-
);
|
|
2355
|
+
), "fitting");
|
|
2356
|
+
const firstGuess = fitting(liveRows);
|
|
2357
|
+
const over = fitting(firstGuess);
|
|
2358
|
+
const under = fitting(over);
|
|
2359
|
+
const recommendedMaxRows = Math.max(firstGuess, under);
|
|
2264
2360
|
const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
|
|
2265
2361
|
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
2266
2362
|
const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
|
|
@@ -2273,15 +2369,20 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2273
2369
|
const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
|
|
2274
2370
|
const containerBound = binding.source === "container memory limit";
|
|
2275
2371
|
const chunked = rowsLive !== null;
|
|
2276
|
-
const
|
|
2277
|
-
|
|
2372
|
+
const chunkHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForChunk(rows) : 0), "chunkHeldBudget");
|
|
2373
|
+
const chunkFitting = /* @__PURE__ */ __name((rows) => recommendedChunkFor(
|
|
2374
|
+
chunkHeldBudget(rows),
|
|
2278
2375
|
symbolTableSize,
|
|
2279
2376
|
maxRows,
|
|
2280
2377
|
totalRows,
|
|
2281
2378
|
columnCount,
|
|
2282
2379
|
liveRowsPerChunk,
|
|
2283
2380
|
includeExternal
|
|
2284
|
-
)
|
|
2381
|
+
), "chunkFitting");
|
|
2382
|
+
const callersChunk = chunked ? Math.max(1, Math.floor(rowsLive / Math.max(1, liveRowsPerChunk))) : 0;
|
|
2383
|
+
const firstChunk = chunked ? chunkFitting(callersChunk) : 0;
|
|
2384
|
+
const overChunk = chunked ? chunkFitting(firstChunk) : 0;
|
|
2385
|
+
const recommendedChunk = chunked ? Math.max(firstChunk, chunkFitting(overChunk)) : 0;
|
|
2285
2386
|
const knob = chunked ? "chunkSize" : "limit";
|
|
2286
2387
|
const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
|
|
2287
2388
|
const nothingFits = recommendedValue === 0;
|
|
@@ -2293,31 +2394,75 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2293
2394
|
} else {
|
|
2294
2395
|
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
2295
2396
|
}
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
{
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
heapLimitMB,
|
|
2305
|
-
reportedHeapLimitMB,
|
|
2306
|
-
availableRamMB,
|
|
2307
|
-
limitingFactor,
|
|
2308
|
-
limitingScope,
|
|
2309
|
-
memoryBudget: budget.candidates,
|
|
2310
|
-
memoryObserved: budget.observed,
|
|
2311
|
-
columnCount,
|
|
2312
|
-
totalRows,
|
|
2313
|
-
maxRows,
|
|
2314
|
-
recommendedMaxRows,
|
|
2315
|
-
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
2316
|
-
// recommendation for the other.
|
|
2317
|
-
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
2397
|
+
const suggestions = [];
|
|
2398
|
+
if (!nothingFits) {
|
|
2399
|
+
suggestions.push({ option: knob, value: recommendedValue });
|
|
2400
|
+
}
|
|
2401
|
+
if (!containerBound) {
|
|
2402
|
+
let needed = Math.ceil(estimatedMemory / safetyFactor / (1024 * 1024));
|
|
2403
|
+
while (Math.max(needed * 1024 * 1024, MINIMUM_BUDGET_BYTES) * safetyFactor < estimatedMemory) {
|
|
2404
|
+
needed += 1;
|
|
2318
2405
|
}
|
|
2319
|
-
|
|
2406
|
+
suggestions.push({ nodeOption: "--max-old-space-size", value: needed });
|
|
2407
|
+
}
|
|
2408
|
+
return {
|
|
2409
|
+
fits: false,
|
|
2410
|
+
reason: "memory",
|
|
2411
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2412
|
+
budget: budgetOf(budget, tightest, safetyFactor),
|
|
2413
|
+
// The symbol term is six times the bytes on disk, an overhead measured across files rather than
|
|
2414
|
+
// derived, so any read with symbols in it is an estimate and says so. Only a read that decodes
|
|
2415
|
+
// nothing can be exact.
|
|
2416
|
+
exact: symbolTableSize === 0,
|
|
2417
|
+
suggestions,
|
|
2418
|
+
refusal: {
|
|
2419
|
+
message: `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
2420
|
+
context: {
|
|
2421
|
+
symbolTableSize,
|
|
2422
|
+
symbolTableSizeMB: sizeMB,
|
|
2423
|
+
estimatedMemoryMB: estimatedMB,
|
|
2424
|
+
availableMemoryMB: availableMB,
|
|
2425
|
+
heapLimitMB,
|
|
2426
|
+
reportedHeapLimitMB,
|
|
2427
|
+
availableRamMB,
|
|
2428
|
+
limitingFactor,
|
|
2429
|
+
limitingScope,
|
|
2430
|
+
memoryBudget: budget.candidates,
|
|
2431
|
+
memoryObserved: budget.observed,
|
|
2432
|
+
columnCount,
|
|
2433
|
+
totalRows,
|
|
2434
|
+
maxRows,
|
|
2435
|
+
recommendedMaxRows,
|
|
2436
|
+
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
2437
|
+
// recommendation for the other.
|
|
2438
|
+
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
};
|
|
2320
2442
|
}
|
|
2443
|
+
return {
|
|
2444
|
+
fits: true,
|
|
2445
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2446
|
+
budget: budgetOf(budget, tightest, safetyFactor),
|
|
2447
|
+
exact: symbolTableSize === 0,
|
|
2448
|
+
suggestions: []
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
function answerOf(answer) {
|
|
2452
|
+
const { refusal, ...rest } = answer;
|
|
2453
|
+
return rest;
|
|
2454
|
+
}
|
|
2455
|
+
function budgetOf(budget, tightest, safetyFactor) {
|
|
2456
|
+
const processLimit = budget.candidates.find((candidate) => candidate.source !== "V8 heap limit");
|
|
2457
|
+
return {
|
|
2458
|
+
heapBytes: usableOldSpaceLimit(),
|
|
2459
|
+
processBytes: processLimit ? processLimit.bytes : null,
|
|
2460
|
+
bound: tightest.heapOnly ? "heap" : "process",
|
|
2461
|
+
safetyFactor,
|
|
2462
|
+
allowedBytes: tightest.allowed ?? tightest.bytes * safetyFactor,
|
|
2463
|
+
candidates: budget.candidates,
|
|
2464
|
+
observed: budget.observed
|
|
2465
|
+
};
|
|
2321
2466
|
}
|
|
2322
2467
|
function formatCount(value) {
|
|
2323
2468
|
return value.toLocaleString("en-US");
|
|
@@ -2339,7 +2484,7 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
2339
2484
|
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
2340
2485
|
);
|
|
2341
2486
|
}
|
|
2342
|
-
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
2487
|
+
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES, noBytesHeld;
|
|
2343
2488
|
var init_memoryUtils = __esm({
|
|
2344
2489
|
"src/util/memoryUtils.js"() {
|
|
2345
2490
|
init_QvdErrors();
|
|
@@ -2357,7 +2502,11 @@ var init_memoryUtils = __esm({
|
|
|
2357
2502
|
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
2358
2503
|
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
2359
2504
|
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
2505
|
+
noBytesHeld = Object.freeze({ held: 0, forRows: /* @__PURE__ */ __name(() => 0, "forRows"), forChunk: /* @__PURE__ */ __name(() => 0, "forChunk") });
|
|
2360
2506
|
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
2507
|
+
__name(checkMemory, "checkMemory");
|
|
2508
|
+
__name(answerOf, "answerOf");
|
|
2509
|
+
__name(budgetOf, "budgetOf");
|
|
2361
2510
|
__name(formatCount, "formatCount");
|
|
2362
2511
|
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
2363
2512
|
}
|
|
@@ -2476,6 +2625,7 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
|
|
|
2476
2625
|
function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
2477
2626
|
const symbolsOffset = headerInteger(field["Offset"]);
|
|
2478
2627
|
const symbolsLength = headerInteger(field["Length"]);
|
|
2628
|
+
const symbolCount = headerInteger(field["NoOfSymbols"]);
|
|
2479
2629
|
if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
|
|
2480
2630
|
throw new exports.QvdCorruptedError("Invalid symbol offset", {
|
|
2481
2631
|
field: field["FieldName"],
|
|
@@ -2492,6 +2642,14 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
|
|
|
2492
2642
|
stage: "parseSymbolTable"
|
|
2493
2643
|
});
|
|
2494
2644
|
}
|
|
2645
|
+
if (isNaN(symbolCount) || !Number.isSafeInteger(symbolCount) || symbolCount < 0) {
|
|
2646
|
+
throw new exports.QvdCorruptedError("Invalid symbol count", {
|
|
2647
|
+
field: field["FieldName"],
|
|
2648
|
+
noOfSymbols: symbolCount,
|
|
2649
|
+
file: filePath,
|
|
2650
|
+
stage: "parseSymbolTable"
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2495
2653
|
if (symbolsOffset + symbolsLength > symbolBufferLength) {
|
|
2496
2654
|
throw new exports.QvdCorruptedError("Symbol data extends beyond buffer", {
|
|
2497
2655
|
field: field["FieldName"],
|
|
@@ -2747,12 +2905,25 @@ var init_validationUtils = __esm({
|
|
|
2747
2905
|
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
2748
2906
|
}
|
|
2749
2907
|
});
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2908
|
+
function nulFinder(area, { reach, rebaseAfter }) {
|
|
2909
|
+
assert4__default.default(reach - rebaseAfter > MAX_TEXT_BYTES, "A text search view must hold the longest text a symbol may have.");
|
|
2910
|
+
if (area.length <= reach) {
|
|
2911
|
+
return (from) => area.indexOf(0, from);
|
|
2912
|
+
}
|
|
2913
|
+
let base = 0;
|
|
2914
|
+
let view = area.subarray(0, reach);
|
|
2915
|
+
return (from) => {
|
|
2916
|
+
if (from - base > rebaseAfter) {
|
|
2917
|
+
base = from;
|
|
2918
|
+
view = area.subarray(base, Math.min(area.length, base + reach));
|
|
2919
|
+
}
|
|
2920
|
+
const found = view.indexOf(0, from - base);
|
|
2921
|
+
return found === -1 ? -1 : base + found;
|
|
2922
|
+
};
|
|
2923
|
+
}
|
|
2924
|
+
function textEnd(findNul, areaEnd, from, kind, fieldName, filePath, base) {
|
|
2925
|
+
const found = findNul(from);
|
|
2926
|
+
if ((found === -1 ? areaEnd : found) - from > MAX_TEXT_BYTES) {
|
|
2756
2927
|
throw new exports.QvdCorruptedError(`${kind} exceeds maximum length`, {
|
|
2757
2928
|
field: fieldName,
|
|
2758
2929
|
maxLength: MAX_TEXT_BYTES,
|
|
@@ -2763,58 +2934,59 @@ function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
|
|
|
2763
2934
|
if (found === -1) {
|
|
2764
2935
|
throw new exports.QvdCorruptedError(`${kind} not null-terminated`, {
|
|
2765
2936
|
field: fieldName,
|
|
2766
|
-
pointer:
|
|
2767
|
-
|
|
2937
|
+
pointer: base + from,
|
|
2938
|
+
areaEnd: base + areaEnd,
|
|
2768
2939
|
file: filePath,
|
|
2769
2940
|
stage: "parseSymbolTable"
|
|
2770
2941
|
});
|
|
2771
2942
|
}
|
|
2772
2943
|
return found;
|
|
2773
2944
|
}
|
|
2774
|
-
function overflow(message, pointer,
|
|
2945
|
+
function overflow(message, pointer, areaEnd, fieldName, filePath, base) {
|
|
2775
2946
|
throw new exports.QvdCorruptedError(message, {
|
|
2776
2947
|
field: fieldName,
|
|
2777
|
-
pointer,
|
|
2778
|
-
|
|
2948
|
+
pointer: base + pointer,
|
|
2949
|
+
areaEnd: base + areaEnd,
|
|
2779
2950
|
file: filePath,
|
|
2780
2951
|
stage: "parseSymbolTable"
|
|
2781
2952
|
});
|
|
2782
2953
|
}
|
|
2783
|
-
function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
|
|
2784
|
-
const
|
|
2954
|
+
function parseFieldSymbols(symbolBuffer, start, end, symbolCount, keep, fieldName, filePath, search = TEXT_SEARCH, base = 0) {
|
|
2955
|
+
const area = symbolBuffer.subarray(0, end);
|
|
2956
|
+
const findNul = nulFinder(area, search);
|
|
2785
2957
|
const numbers = [];
|
|
2786
2958
|
const texts = [];
|
|
2787
2959
|
let pointer = start;
|
|
2788
2960
|
while (pointer < end) {
|
|
2789
|
-
const typeByte =
|
|
2961
|
+
const typeByte = area[pointer++];
|
|
2790
2962
|
const decode = keep === null || keep.has(numbers.length);
|
|
2791
2963
|
let number = null;
|
|
2792
2964
|
let text = null;
|
|
2793
2965
|
switch (typeByte) {
|
|
2794
2966
|
case 1: {
|
|
2967
|
+
if (pointer + 4 > end) {
|
|
2968
|
+
overflow("Buffer overflow reading integer symbol", pointer, end, fieldName, filePath, base);
|
|
2969
|
+
}
|
|
2795
2970
|
if (decode) {
|
|
2796
|
-
|
|
2797
|
-
overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
|
|
2798
|
-
}
|
|
2799
|
-
number = symbolBuffer.readInt32LE(pointer);
|
|
2971
|
+
number = area.readInt32LE(pointer);
|
|
2800
2972
|
}
|
|
2801
2973
|
pointer += 4;
|
|
2802
2974
|
break;
|
|
2803
2975
|
}
|
|
2804
2976
|
case 2: {
|
|
2977
|
+
if (pointer + 8 > end) {
|
|
2978
|
+
overflow("Buffer overflow reading double symbol", pointer, end, fieldName, filePath, base);
|
|
2979
|
+
}
|
|
2805
2980
|
if (decode) {
|
|
2806
|
-
|
|
2807
|
-
overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
|
|
2808
|
-
}
|
|
2809
|
-
number = symbolBuffer.readDoubleLE(pointer);
|
|
2981
|
+
number = area.readDoubleLE(pointer);
|
|
2810
2982
|
}
|
|
2811
2983
|
pointer += 8;
|
|
2812
2984
|
break;
|
|
2813
2985
|
}
|
|
2814
2986
|
case 4: {
|
|
2815
|
-
const terminator = textEnd(
|
|
2987
|
+
const terminator = textEnd(findNul, end, pointer, "String symbol", fieldName, filePath, base);
|
|
2816
2988
|
if (decode) {
|
|
2817
|
-
text =
|
|
2989
|
+
text = area.toString("utf8", pointer, terminator);
|
|
2818
2990
|
}
|
|
2819
2991
|
pointer = terminator + 1;
|
|
2820
2992
|
break;
|
|
@@ -2822,14 +2994,22 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2822
2994
|
case 5:
|
|
2823
2995
|
case 6: {
|
|
2824
2996
|
const numberBytes = typeByte === 5 ? 4 : 8;
|
|
2825
|
-
if (pointer + numberBytes >
|
|
2826
|
-
const read =
|
|
2827
|
-
overflow(`Buffer overflow reading ${read}`, pointer,
|
|
2997
|
+
if (pointer + numberBytes > end) {
|
|
2998
|
+
const read = typeByte === 5 ? "dual integer symbol" : "dual double symbol";
|
|
2999
|
+
overflow(`Buffer overflow reading ${read}`, pointer, end, fieldName, filePath, base);
|
|
2828
3000
|
}
|
|
2829
|
-
const terminator = textEnd(
|
|
3001
|
+
const terminator = textEnd(
|
|
3002
|
+
findNul,
|
|
3003
|
+
end,
|
|
3004
|
+
pointer + numberBytes,
|
|
3005
|
+
"Dual string symbol",
|
|
3006
|
+
fieldName,
|
|
3007
|
+
filePath,
|
|
3008
|
+
base
|
|
3009
|
+
);
|
|
2830
3010
|
if (decode) {
|
|
2831
|
-
number = typeByte === 5 ?
|
|
2832
|
-
text =
|
|
3011
|
+
number = typeByte === 5 ? area.readInt32LE(pointer) : area.readDoubleLE(pointer);
|
|
3012
|
+
text = area.toString("utf8", pointer + numberBytes, terminator);
|
|
2833
3013
|
}
|
|
2834
3014
|
pointer = terminator + 1;
|
|
2835
3015
|
break;
|
|
@@ -2837,7 +3017,7 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2837
3017
|
default: {
|
|
2838
3018
|
throw new exports.QvdParseError("Unknown symbol type byte", {
|
|
2839
3019
|
typeByte: typeByte.toString(16),
|
|
2840
|
-
offset: pointer - 1,
|
|
3020
|
+
offset: base + pointer - 1,
|
|
2841
3021
|
file: filePath,
|
|
2842
3022
|
stage: "parseSymbolTable"
|
|
2843
3023
|
});
|
|
@@ -2846,16 +3026,28 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2846
3026
|
numbers.push(number);
|
|
2847
3027
|
texts.push(text);
|
|
2848
3028
|
}
|
|
3029
|
+
assert4__default.default(pointer === end, `The symbols of ${fieldName} were walked to byte ${pointer} of an area ending at ${end}.`);
|
|
3030
|
+
if (numbers.length !== symbolCount) {
|
|
3031
|
+
throw new exports.QvdCorruptedError("Symbol count mismatch", {
|
|
3032
|
+
field: fieldName,
|
|
3033
|
+
symbolCount: numbers.length,
|
|
3034
|
+
noOfSymbols: symbolCount,
|
|
3035
|
+
file: filePath,
|
|
3036
|
+
stage: "parseSymbolTable"
|
|
3037
|
+
});
|
|
3038
|
+
}
|
|
2849
3039
|
return { numbers, texts };
|
|
2850
3040
|
}
|
|
2851
|
-
function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
|
|
2852
|
-
return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
|
|
3041
|
+
function countFieldSymbols(symbolBuffer, start, end, symbolCount, fieldName, filePath, base = 0) {
|
|
3042
|
+
return parseFieldSymbols(symbolBuffer, start, end, symbolCount, DECODE_NOTHING, fieldName, filePath, void 0, base).numbers.length;
|
|
2853
3043
|
}
|
|
2854
|
-
var MAX_TEXT_BYTES, DECODE_NOTHING;
|
|
3044
|
+
var MAX_TEXT_BYTES, TEXT_SEARCH, DECODE_NOTHING;
|
|
2855
3045
|
var init_symbolParser = __esm({
|
|
2856
3046
|
"src/util/symbolParser.js"() {
|
|
2857
3047
|
init_QvdErrors();
|
|
2858
3048
|
MAX_TEXT_BYTES = 1048576;
|
|
3049
|
+
TEXT_SEARCH = Object.freeze({ reach: 2 ** 31 - 1, rebaseAfter: 2 ** 30 });
|
|
3050
|
+
__name(nulFinder, "nulFinder");
|
|
2859
3051
|
__name(textEnd, "textEnd");
|
|
2860
3052
|
__name(overflow, "overflow");
|
|
2861
3053
|
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
@@ -3399,7 +3591,17 @@ async function parseHeaderXml(text, file, stage) {
|
|
|
3399
3591
|
}
|
|
3400
3592
|
return parsed;
|
|
3401
3593
|
}
|
|
3402
|
-
|
|
3594
|
+
function symbolBytesOf(selected, symbolTableLength) {
|
|
3595
|
+
const areaBytes = selected.map((field) => headerInteger(field["Length"]));
|
|
3596
|
+
return areaBytes.every((bytes) => Number.isSafeInteger(bytes) && bytes >= 0) ? Math.min(
|
|
3597
|
+
symbolTableLength,
|
|
3598
|
+
areaBytes.reduce((sum, bytes) => sum + bytes, 0)
|
|
3599
|
+
) : symbolTableLength;
|
|
3600
|
+
}
|
|
3601
|
+
function readPasses(analysisAhead) {
|
|
3602
|
+
return analysisAhead ? 2 : 1;
|
|
3603
|
+
}
|
|
3604
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, SLICE_BYTES, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
|
|
3403
3605
|
var init_QvdFileReader = __esm({
|
|
3404
3606
|
"src/QvdFileReader.js"() {
|
|
3405
3607
|
init_QvdDataFrame();
|
|
@@ -3417,9 +3619,12 @@ var init_QvdFileReader = __esm({
|
|
|
3417
3619
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
3418
3620
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
3419
3621
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
3622
|
+
SLICE_BYTES = 16 * 1024 * 1024;
|
|
3420
3623
|
COUNT_SYMBOLS_PAST = 65536;
|
|
3421
3624
|
__name(chunksFrom, "chunksFrom");
|
|
3422
3625
|
__name(parseHeaderXml, "parseHeaderXml");
|
|
3626
|
+
__name(symbolBytesOf, "symbolBytesOf");
|
|
3627
|
+
__name(readPasses, "readPasses");
|
|
3423
3628
|
exports.QvdFileReader = class {
|
|
3424
3629
|
static {
|
|
3425
3630
|
__name(this, "QvdFileReader");
|
|
@@ -3448,6 +3653,10 @@ var init_QvdFileReader = __esm({
|
|
|
3448
3653
|
* above which a lazy load switches to the two-pass filtering path. The default of 50MB is
|
|
3449
3654
|
* the point where the extra analysis pass pays for itself; lower it to use filtering on
|
|
3450
3655
|
* smaller files, raise it to keep the simpler single-pass read for longer.
|
|
3656
|
+
* @param {number} [options.sliceBytes=16777216] The most bytes of records a read holds at a time. An
|
|
3657
|
+
* option rather than a constant for the reason `symbolFilteringThreshold` is one: so that a test can
|
|
3658
|
+
* cross the boundaries between slices in a small file. There is no other reason to change it, and it
|
|
3659
|
+
* is not one of the options a read through `QvdDataFrame` or `QvdColumnTable` passes on.
|
|
3451
3660
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
3452
3661
|
* appear. Null reads every field, in file order. An unknown or repeated name is refused.
|
|
3453
3662
|
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
@@ -3472,6 +3681,7 @@ var init_QvdFileReader = __esm({
|
|
|
3472
3681
|
allowedDir,
|
|
3473
3682
|
memorySafetyFactor = 0.8,
|
|
3474
3683
|
symbolFilteringThreshold = 50 * 1024 * 1024,
|
|
3684
|
+
sliceBytes = SLICE_BYTES,
|
|
3475
3685
|
materialisesRows = true,
|
|
3476
3686
|
fields = null,
|
|
3477
3687
|
duals,
|
|
@@ -3487,6 +3697,14 @@ var init_QvdFileReader = __esm({
|
|
|
3487
3697
|
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
3488
3698
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
3489
3699
|
this._symbolFilteringThreshold = symbolFilteringThreshold;
|
|
3700
|
+
if (!Number.isSafeInteger(sliceBytes) || sliceBytes <= 0) {
|
|
3701
|
+
throw new exports.QvdValidationError("sliceBytes must be a positive integer", {
|
|
3702
|
+
provided: sliceBytes,
|
|
3703
|
+
type: typeof sliceBytes,
|
|
3704
|
+
file: this._path
|
|
3705
|
+
});
|
|
3706
|
+
}
|
|
3707
|
+
this._sliceBytes = sliceBytes;
|
|
3490
3708
|
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
3491
3709
|
throw new exports.QvdValidationError("onProgress must be a function", {
|
|
3492
3710
|
provided: onProgress,
|
|
@@ -3504,7 +3722,11 @@ var init_QvdFileReader = __esm({
|
|
|
3504
3722
|
this._requestedFields = fields === void 0 ? null : fields;
|
|
3505
3723
|
this._onProgress = onProgress;
|
|
3506
3724
|
this._signal = signal;
|
|
3507
|
-
this.
|
|
3725
|
+
this._headerBuffer = null;
|
|
3726
|
+
this._handle = null;
|
|
3727
|
+
this._failed = null;
|
|
3728
|
+
this._reading = false;
|
|
3729
|
+
this._symbolAreas = null;
|
|
3508
3730
|
this._headerOffset = null;
|
|
3509
3731
|
this._symbolTableOffset = null;
|
|
3510
3732
|
this._indexTableOffset = null;
|
|
@@ -3515,7 +3737,6 @@ var init_QvdFileReader = __esm({
|
|
|
3515
3737
|
this._symbolTable = null;
|
|
3516
3738
|
this._indexColumns = null;
|
|
3517
3739
|
this._rowsDecoded = 0;
|
|
3518
|
-
this._bufferFirstRow = 0;
|
|
3519
3740
|
this._fileSize = null;
|
|
3520
3741
|
this._headerMatchesFile = false;
|
|
3521
3742
|
}
|
|
@@ -3554,54 +3775,137 @@ var init_QvdFileReader = __esm({
|
|
|
3554
3775
|
}
|
|
3555
3776
|
}
|
|
3556
3777
|
/**
|
|
3557
|
-
*
|
|
3558
|
-
*
|
|
3559
|
-
*
|
|
3560
|
-
*
|
|
3561
|
-
*
|
|
3562
|
-
*
|
|
3563
|
-
*
|
|
3564
|
-
*
|
|
3565
|
-
*
|
|
3566
|
-
*
|
|
3567
|
-
*
|
|
3568
|
-
*
|
|
3569
|
-
*
|
|
3570
|
-
*
|
|
3571
|
-
*
|
|
3572
|
-
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
3573
|
-
* 4. Read only those calculated bytes, by position
|
|
3574
|
-
* 5. Rest of parsing proceeds normally with limited data
|
|
3575
|
-
*
|
|
3576
|
-
* WHY THIS APPROACH:
|
|
3577
|
-
* - Symbol table must be fully loaded (contains all unique values)
|
|
3578
|
-
* - Index table can be partially loaded (only rows we need)
|
|
3579
|
-
* - Reading chunks to find the header is efficient for unknown header sizes
|
|
3580
|
-
* - Direct byte-range reading for remaining data is fastest
|
|
3778
|
+
* Opens the file and reads its header, and for a read of rows checks that the read can be made.
|
|
3779
|
+
*
|
|
3780
|
+
* Nothing past the header is read here. The selected fields' symbols and the records are read by position
|
|
3781
|
+
* as they are parsed - see `_symbolAreaOf` and `_forEachSlice` - so no read holds the file in one buffer
|
|
3782
|
+
* (#122). A read of rows therefore leaves the file open, and whatever started the read closes it with
|
|
3783
|
+
* `_closeFile` once the last record it needs is decoded; a header-only read closes it here.
|
|
3784
|
+
*
|
|
3785
|
+
* A windowed read - anything with `offset`, `limit` or `maxRows` - reads the header, the symbols of the
|
|
3786
|
+
* fields it selects, and the window's records, and no byte between. Measured on
|
|
3787
|
+
* `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows over 20 fields: the last thousand rows take 19 ms
|
|
3788
|
+
* against 636 ms for the whole file. A selected field's area is read in full whatever the window,
|
|
3789
|
+
* because a stored index in any row can address any of that field's symbols; the areas of the fields
|
|
3790
|
+
* `fields` leaves out are not read at all. So a window's gain scales with how much of the file is
|
|
3791
|
+
* rows: on a file whose bytes are mostly distinct values of the fields it reads there is very little
|
|
3792
|
+
* to save, which is what `symbolFilteringThreshold` and the two-pass path exist for.
|
|
3581
3793
|
*
|
|
3582
3794
|
* All of it goes through one handle, opened once, on the file the containment check approved. See
|
|
3583
3795
|
* `chunksFrom` and `openChecked` for why a read no longer opens the path more than once.
|
|
3584
3796
|
*
|
|
3585
|
-
* A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
|
|
3586
|
-
* table from the front of the file, and the window's records from wherever they sit. The bytes
|
|
3587
|
-
* between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
|
|
3588
|
-
* fixture a 0.4MB read rather than a 38MB one.
|
|
3589
|
-
*
|
|
3590
3797
|
* @param {QvdRowWindow} window The rows to read.
|
|
3591
|
-
* @param {boolean} [headerOnly=false] Stop once the XML header has been read,
|
|
3592
|
-
*
|
|
3593
|
-
*
|
|
3594
|
-
* for a 40GB one.
|
|
3798
|
+
* @param {boolean} [headerOnly=false] Stop once the XML header has been read, and close the file. This
|
|
3799
|
+
* is the metadata-only path: the header is a few kilobytes whatever the file's size, so reading a
|
|
3800
|
+
* schema costs the same for a 40MB file as for a 40GB one.
|
|
3595
3801
|
* @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
|
|
3596
3802
|
* that is fewer than the window covers - see `_prepare`.
|
|
3597
3803
|
* @private
|
|
3598
3804
|
*/
|
|
3599
3805
|
async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
|
|
3806
|
+
assert4__default.default(this._reading, "A read opens the QVD file only once it has started, through _startRead.");
|
|
3807
|
+
this._symbolTable = null;
|
|
3808
|
+
this._indexColumns = null;
|
|
3809
|
+
this._rowsDecoded = 0;
|
|
3600
3810
|
this._throwIfAborted();
|
|
3601
3811
|
this._emitProgress("read", 0, 1);
|
|
3602
3812
|
const failed = rethrowAsIoError(this._path, "read");
|
|
3603
3813
|
const handle = await openChecked(checkPath(this._path, this._allowedDir), "read", failed);
|
|
3604
|
-
|
|
3814
|
+
if (headerOnly) {
|
|
3815
|
+
await closeAfter(handle, failed, () => this._readFrom(handle, window, true, liveRows, failed));
|
|
3816
|
+
return;
|
|
3817
|
+
}
|
|
3818
|
+
this._handle = handle;
|
|
3819
|
+
this._failed = failed;
|
|
3820
|
+
try {
|
|
3821
|
+
await this._readFrom(handle, window, false, liveRows, failed);
|
|
3822
|
+
} catch (error) {
|
|
3823
|
+
await this._closeFile(true);
|
|
3824
|
+
throw error;
|
|
3825
|
+
}
|
|
3826
|
+
}
|
|
3827
|
+
/**
|
|
3828
|
+
* Starts a read on this reader, refusing it while another is under way.
|
|
3829
|
+
*
|
|
3830
|
+
* A read of rows holds the file, and what it has read of it, on the reader until it ends. A second read
|
|
3831
|
+
* started meanwhile - `load()` while an iteration is suspended, say - would take over that state and
|
|
3832
|
+
* leave the first read's file open. Reads one after another are fine. Called before a read takes charge
|
|
3833
|
+
* of closing the file, so that refusing the second read cannot close the first one's.
|
|
3834
|
+
*
|
|
3835
|
+
* The first thing every read does, and synchronous: the flag is set before the read's first `await`, so
|
|
3836
|
+
* two reads started together - `Promise.all([reader.load(), reader.load()])` - cannot both pass. The check
|
|
3837
|
+
* used to be of the handle, which is set only once the file has opened, and both did: the second read's
|
|
3838
|
+
* handle replaced the first's, which was never closed, and the first read to finish closed the file the
|
|
3839
|
+
* other was still reading.
|
|
3840
|
+
*
|
|
3841
|
+
* @throws {QvdValidationError} If a read is under way.
|
|
3842
|
+
* @private
|
|
3843
|
+
*/
|
|
3844
|
+
_startRead() {
|
|
3845
|
+
if (this._reading) {
|
|
3846
|
+
throw new exports.QvdValidationError("The reader is already reading this file: finish that read first", {
|
|
3847
|
+
file: this._path
|
|
3848
|
+
});
|
|
3849
|
+
}
|
|
3850
|
+
this._reading = true;
|
|
3851
|
+
}
|
|
3852
|
+
/**
|
|
3853
|
+
* Ends the read `_startRead` began: closes its file, if it still holds one, and lets the next read start.
|
|
3854
|
+
*
|
|
3855
|
+
* @param {boolean} failing Whether the read is already throwing - see `_closeFile`.
|
|
3856
|
+
* @private
|
|
3857
|
+
*/
|
|
3858
|
+
async _endRead(failing) {
|
|
3859
|
+
try {
|
|
3860
|
+
await this._closeFile(failing);
|
|
3861
|
+
} finally {
|
|
3862
|
+
this._reading = false;
|
|
3863
|
+
}
|
|
3864
|
+
}
|
|
3865
|
+
/**
|
|
3866
|
+
* Closes the file a read of rows opened, and drops what it had read of it.
|
|
3867
|
+
*
|
|
3868
|
+
* The rule `closeAfter` follows: a close that fails is reported only when the read succeeded, so it can
|
|
3869
|
+
* never replace the error that says what went wrong. After a successful read it is the only news.
|
|
3870
|
+
*
|
|
3871
|
+
* @param {boolean} failing Whether the read is already throwing.
|
|
3872
|
+
* @private
|
|
3873
|
+
*/
|
|
3874
|
+
async _closeFile(failing) {
|
|
3875
|
+
const handle = this._handle;
|
|
3876
|
+
const failed = this._failed;
|
|
3877
|
+
this._handle = null;
|
|
3878
|
+
this._failed = null;
|
|
3879
|
+
this._symbolAreas = null;
|
|
3880
|
+
if (handle === null || failed === null) {
|
|
3881
|
+
return;
|
|
3882
|
+
}
|
|
3883
|
+
if (failing) {
|
|
3884
|
+
await handle.close().catch(() => {
|
|
3885
|
+
});
|
|
3886
|
+
return;
|
|
3887
|
+
}
|
|
3888
|
+
await handle.close().catch(failed);
|
|
3889
|
+
}
|
|
3890
|
+
/**
|
|
3891
|
+
* Runs a read, the only one under way on this reader, and closes its file when it ends, however it ends.
|
|
3892
|
+
*
|
|
3893
|
+
* @template T
|
|
3894
|
+
* @param {() => Promise<T>} read The read, from opening the file to its last record.
|
|
3895
|
+
* @return {Promise<T>} What it returned.
|
|
3896
|
+
* @private
|
|
3897
|
+
*/
|
|
3898
|
+
async _closingAfter(read) {
|
|
3899
|
+
this._startRead();
|
|
3900
|
+
let result;
|
|
3901
|
+
try {
|
|
3902
|
+
result = await read();
|
|
3903
|
+
} catch (error) {
|
|
3904
|
+
await this._endRead(true);
|
|
3905
|
+
throw error;
|
|
3906
|
+
}
|
|
3907
|
+
await this._endRead(false);
|
|
3908
|
+
return result;
|
|
3605
3909
|
}
|
|
3606
3910
|
/**
|
|
3607
3911
|
* Reads what `_readData` was asked for, through the handle it opened.
|
|
@@ -3662,51 +3966,61 @@ var init_QvdFileReader = __esm({
|
|
|
3662
3966
|
const indexTableOffset = symbolTableOffset + symbolTableLength;
|
|
3663
3967
|
const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
|
|
3664
3968
|
const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
return;
|
|
3669
|
-
}
|
|
3670
|
-
const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
|
|
3969
|
+
const { size: fileSize } = await handle.stat().catch(failed);
|
|
3970
|
+
this._fileSize = fileSize;
|
|
3971
|
+
this._headerMatchesFile = false;
|
|
3671
3972
|
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
3672
3973
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3673
3974
|
);
|
|
3674
3975
|
if (headerNumbersUsable) {
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3976
|
+
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
|
|
3977
|
+
}
|
|
3978
|
+
if (headerOnly) {
|
|
3979
|
+
this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3980
|
+
this._emitProgress("read", 1, 1);
|
|
3981
|
+
return;
|
|
3678
3982
|
}
|
|
3983
|
+
this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3984
|
+
const selected = selectFields(headerFields, this._requestedFields, this._path);
|
|
3985
|
+
const columnCount = selected.length;
|
|
3986
|
+
const symbolBytes = symbolBytesOf(selected, symbolTableLength);
|
|
3679
3987
|
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3680
3988
|
const windowRows = resolved.limit;
|
|
3681
3989
|
if (headerNumbersUsable && this._headerMatchesFile) {
|
|
3682
3990
|
validateMemoryAvailability(
|
|
3683
|
-
|
|
3991
|
+
symbolBytes,
|
|
3684
3992
|
windowRows,
|
|
3685
3993
|
totalRows,
|
|
3686
3994
|
this._path,
|
|
3687
3995
|
this._memorySafetyFactor,
|
|
3688
3996
|
columnCount,
|
|
3689
3997
|
this._materialisesRows,
|
|
3690
|
-
liveRows
|
|
3998
|
+
liveRows,
|
|
3999
|
+
this._bytesHeld(
|
|
4000
|
+
symbolBytes,
|
|
4001
|
+
windowRows,
|
|
4002
|
+
recordSize,
|
|
4003
|
+
liveRows,
|
|
4004
|
+
this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)
|
|
4005
|
+
),
|
|
4006
|
+
// What it reads, which is not what it holds - the records go through one buffer and are not
|
|
4007
|
+
// kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
|
|
4008
|
+
//
|
|
4009
|
+
// Twice over where the symbol-usage pass is still ahead of it: that pass reads the window's
|
|
4010
|
+
// records to find which symbols the rows use, and the decode then reads them again. Counted
|
|
4011
|
+
// once, the figure understated the I/O of exactly the reads that do the most of it.
|
|
4012
|
+
symbolBytes + readPasses(this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) * windowRows * recordSize
|
|
3691
4013
|
);
|
|
3692
4014
|
}
|
|
3693
4015
|
if (window.offset === 0 && window.limit === null) {
|
|
3694
|
-
this._buffer = await handle.readFile().catch(failed);
|
|
3695
|
-
this._fileSize = this._buffer.length;
|
|
3696
|
-
this._bufferFirstRow = 0;
|
|
3697
4016
|
this._emitProgress("read", 1, 1);
|
|
3698
4017
|
return;
|
|
3699
4018
|
}
|
|
3700
4019
|
const rowsToLoad = windowRows;
|
|
3701
|
-
validateSymbolTableSizeEarly(
|
|
4020
|
+
validateSymbolTableSizeEarly(symbolBytes, this._path);
|
|
3702
4021
|
validateRecordSize(recordSize, this._path, "readData");
|
|
3703
4022
|
validateRecordCount(totalRows, this._path, "readData");
|
|
3704
|
-
const
|
|
3705
|
-
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
3706
|
-
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
3707
|
-
const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
|
|
3708
|
-
const { size: fileSize } = await handle.stat().catch(failed);
|
|
3709
|
-
this._fileSize = fileSize;
|
|
4023
|
+
const fileBytesRequired = indexTableOffset + (resolved.offset + rowsToLoad) * recordSize;
|
|
3710
4024
|
if (fileBytesRequired > fileSize) {
|
|
3711
4025
|
throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
|
|
3712
4026
|
file: this._path,
|
|
@@ -3715,47 +4029,35 @@ var init_QvdFileReader = __esm({
|
|
|
3715
4029
|
stage: "readData"
|
|
3716
4030
|
});
|
|
3717
4031
|
}
|
|
3718
|
-
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
3719
|
-
await this._readRange(handle, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3720
|
-
if (indexTableBytesToRead > 0) {
|
|
3721
|
-
await this._readRange(
|
|
3722
|
-
handle,
|
|
3723
|
-
indexTableOffset,
|
|
3724
|
-
indexTableBytesToRead,
|
|
3725
|
-
indexTableOffset + skippedIndexBytes,
|
|
3726
|
-
fileSize,
|
|
3727
|
-
fileBytesRequired
|
|
3728
|
-
);
|
|
3729
|
-
}
|
|
3730
|
-
this._bufferFirstRow = resolved.offset;
|
|
3731
4032
|
this._emitProgress("read", 1, 1);
|
|
3732
4033
|
}
|
|
3733
4034
|
/**
|
|
3734
|
-
* Reads one byte range of the file into
|
|
4035
|
+
* Reads one byte range of the open file into a buffer.
|
|
3735
4036
|
*
|
|
3736
4037
|
* Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
|
|
3737
4038
|
* 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
|
|
3738
4039
|
* try/catch can intercept.
|
|
3739
4040
|
*
|
|
3740
|
-
* @param {
|
|
3741
|
-
* @param {number}
|
|
4041
|
+
* @param {Buffer} target The buffer to read into.
|
|
4042
|
+
* @param {number} targetOffset Where in it to write.
|
|
3742
4043
|
* @param {number} byteCount How many bytes to read.
|
|
3743
4044
|
* @param {number} filePosition Where in the file to read from.
|
|
3744
|
-
* @param {number}
|
|
3745
|
-
* @
|
|
4045
|
+
* @param {number} requiredBytes How far into the file the read has to reach, for the error.
|
|
4046
|
+
* @throws {QvdCorruptedError} If the file ends before the range does.
|
|
3746
4047
|
* @private
|
|
3747
4048
|
*/
|
|
3748
|
-
async
|
|
3749
|
-
|
|
3750
|
-
const
|
|
4049
|
+
async _readAt(target, targetOffset, byteCount, filePosition, requiredBytes) {
|
|
4050
|
+
assert4__default.default(this._handle && this._failed, "The QVD file is not open.");
|
|
4051
|
+
const handle = this._handle;
|
|
4052
|
+
const failed = this._failed;
|
|
3751
4053
|
let done = 0;
|
|
3752
4054
|
while (done < byteCount) {
|
|
3753
4055
|
const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
|
|
3754
|
-
const { bytesRead } = await
|
|
4056
|
+
const { bytesRead } = await handle.read(target, targetOffset + done, length, filePosition + done).catch(failed);
|
|
3755
4057
|
if (bytesRead === 0) {
|
|
3756
4058
|
throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
|
|
3757
4059
|
file: this._path,
|
|
3758
|
-
fileSize,
|
|
4060
|
+
fileSize: this._fileSize,
|
|
3759
4061
|
// Two numbers, because they stopped being the same one when a window began reading two
|
|
3760
4062
|
// ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
|
|
3761
4063
|
// file it gave up. Reporting the position under the name of the count made a windowed
|
|
@@ -3770,12 +4072,279 @@ var init_QvdFileReader = __esm({
|
|
|
3770
4072
|
done += bytesRead;
|
|
3771
4073
|
}
|
|
3772
4074
|
}
|
|
4075
|
+
/**
|
|
4076
|
+
* Bytes of the file a read holds outside the heap while it works, beside the codes the guard counts for
|
|
4077
|
+
* itself: the symbol areas it reads, and the one buffer its records come through.
|
|
4078
|
+
*
|
|
4079
|
+
* The areas are counted whole, although the read lets each range go once its fields are parsed, because
|
|
4080
|
+
* two ranges are both live when a field of one is parsed between two fields of the other - the order the
|
|
4081
|
+
* caller asked for the fields decides it, so the sum is what holds in every order. The slice is the
|
|
4082
|
+
* buffer `_forEachSlice` will allocate, sized by `_sliceRowsFor` so that the charge and the allocation
|
|
4083
|
+
* are one expression rather than two that agree today.
|
|
4084
|
+
*
|
|
4085
|
+
* @param {number} symbolBytes Bytes of symbols the read will read.
|
|
4086
|
+
* @param {number} rows Records it will read.
|
|
4087
|
+
* @param {number} recordSize Bytes per record.
|
|
4088
|
+
* @return {number} Bytes.
|
|
4089
|
+
* @private
|
|
4090
|
+
*/
|
|
4091
|
+
_bytesHeldBy(symbolBytes, rows, recordSize) {
|
|
4092
|
+
const usable = Number.isSafeInteger(rows) && Number.isSafeInteger(recordSize) && rows > 0 && recordSize > 0;
|
|
4093
|
+
return symbolBytes + (usable ? this._sliceRowsFor(rows, recordSize) * recordSize : 0);
|
|
4094
|
+
}
|
|
4095
|
+
/**
|
|
4096
|
+
* Records the one buffer holds while a read of `rowCount` records goes through it.
|
|
4097
|
+
*
|
|
4098
|
+
* A slice is `sliceBytes` of records, rounded down to a whole record, or every record the read has left
|
|
4099
|
+
* when that is fewer - and at least one, since a read of a record wider than `sliceBytes` still has to
|
|
4100
|
+
* hold that record. The single definition: `_forEachSlice` allocates from it and the memory guard is
|
|
4101
|
+
* charged from it, so a change to how a read slices cannot leave the guard pricing the old rule.
|
|
4102
|
+
*
|
|
4103
|
+
* @param {number} rowCount Records the read will read.
|
|
4104
|
+
* @param {number} recordSize Bytes per record.
|
|
4105
|
+
* @return {number} Records in one slice.
|
|
4106
|
+
* @private
|
|
4107
|
+
*/
|
|
4108
|
+
_sliceRowsFor(rowCount, recordSize) {
|
|
4109
|
+
return Math.max(1, Math.min(rowCount, Math.floor(this._sliceBytes / Math.max(1, recordSize))));
|
|
4110
|
+
}
|
|
4111
|
+
/**
|
|
4112
|
+
* What a read holds in bytes of the file, for the memory guard: what it holds now, and what a read
|
|
4113
|
+
* following either piece of advice a refusal can carry would hold instead.
|
|
4114
|
+
*
|
|
4115
|
+
* The two knobs are not the same knob, which is why there are two functions rather than one. A smaller
|
|
4116
|
+
* `limit` is a smaller window, so every record the read touches is one of fewer - the symbol-usage pass
|
|
4117
|
+
* included, since it reads the window. A smaller `chunkSize` leaves the window exactly where it is and
|
|
4118
|
+
* only changes how much of it is decoded at a time, so a read with that pass still ahead of it holds the
|
|
4119
|
+
* window's slice however small the chunk. Priced with `forRows`, such a chunk was charged for the records
|
|
4120
|
+
* of one chunk and then held sixteen megabytes more than that.
|
|
4121
|
+
*
|
|
4122
|
+
* @param {number} symbolBytes Bytes of symbols the read will read.
|
|
4123
|
+
* @param {number} windowRows Rows the read covers.
|
|
4124
|
+
* @param {number} recordSize Bytes per record.
|
|
4125
|
+
* @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
|
|
4126
|
+
* @param {boolean} analysisAhead Whether the symbol-usage pass has still to run.
|
|
4127
|
+
* @return {{held: number, forRows: (rows: number) => number, forChunk: (rows: number) => number}} What
|
|
4128
|
+
* this read holds, what a read of so many rows would hold, and what one reading so many rows a chunk
|
|
4129
|
+
* would hold.
|
|
4130
|
+
* @private
|
|
4131
|
+
*/
|
|
4132
|
+
_bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead) {
|
|
4133
|
+
return {
|
|
4134
|
+
held: this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
|
|
4135
|
+
// A window of so many rows reads so many records at a time, and the pass that reads it ahead of the
|
|
4136
|
+
// decode reads the same rows, so the buffer is sized from the rows either way.
|
|
4137
|
+
forRows: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, rows, recordSize), "forRows"),
|
|
4138
|
+
// A chunk of so many rows, over this read's window - which is what `chunkSize` changes and what it
|
|
4139
|
+
// leaves alone. `_recordsAtOnce` is what answers that, given a chunk size as the rows held at once.
|
|
4140
|
+
forChunk: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, { rows, perChunk: 1 }, analysisAhead), recordSize), "forChunk")
|
|
4141
|
+
};
|
|
4142
|
+
}
|
|
4143
|
+
/**
|
|
4144
|
+
* Whether a read takes the symbol-usage pass, which reads the window's records before the symbol table
|
|
4145
|
+
* is parsed and so before the first chunk is built.
|
|
4146
|
+
*
|
|
4147
|
+
* The one statement of the condition. `_prepare` asks it to decide, and `_readData` asks it before the
|
|
4148
|
+
* header has been parsed, to know what to charge the memory guard: a read with the pass ahead of it
|
|
4149
|
+
* holds a whole slice of records, and a read without it holds only what it reads at a time. Said in two
|
|
4150
|
+
* places, the two would drift and a read would be charged for one path and take the other - which fails
|
|
4151
|
+
* open for a chunked read, and that is the direction the guard exists to prevent.
|
|
4152
|
+
*
|
|
4153
|
+
* Any window that does not cover the whole file is a candidate, which includes one bounded by its offset
|
|
4154
|
+
* rather than by its limit. Covering every row rules it out, however the window was spelled -
|
|
4155
|
+
* `{offset: 0, limit: n}` over all n rows, or an `iterate` of them. Such a read needs every symbol any
|
|
4156
|
+
* row uses, which is what `estimateMemoryUsage` assumes for a full read as well, so the pass has nothing
|
|
4157
|
+
* to filter. It is not free: since the records are read as they are decoded rather than held in one
|
|
4158
|
+
* buffer, the pass reads the window's records and the decode then reads them again. A window that really
|
|
4159
|
+
* is a window pays that for the symbols it saves parsing; a window that is a full read in disguise paid
|
|
4160
|
+
* it for nothing.
|
|
4161
|
+
*
|
|
4162
|
+
* The threshold is an option rather than a constant so this path can be exercised with a small fixture:
|
|
4163
|
+
* it is the most intricate code in the reader, and the only files large enough to reach the 50MB default
|
|
4164
|
+
* are ones no repository should be carrying around. It measures the whole table rather than the areas a
|
|
4165
|
+
* read selects, because what the pass saves is parsing work across the table.
|
|
4166
|
+
*
|
|
4167
|
+
* @param {QvdRowWindow} window The window as the caller spelled it.
|
|
4168
|
+
* @param {{offset: number, limit: number}} resolved Where it lands in this file.
|
|
4169
|
+
* @param {number} totalRows Rows the file declares.
|
|
4170
|
+
* @param {number} symbolTableLength The symbol table's declared length.
|
|
4171
|
+
* @return {boolean} Whether the pass will run.
|
|
4172
|
+
* @private
|
|
4173
|
+
*/
|
|
4174
|
+
_analysisWouldRun(window, resolved, totalRows, symbolTableLength) {
|
|
4175
|
+
return resolved.limit < totalRows && (window.limit !== null || window.offset > 0) && symbolTableLength > this._symbolFilteringThreshold;
|
|
4176
|
+
}
|
|
4177
|
+
/**
|
|
4178
|
+
* Records a read holds at one time, which is what its record buffer is sized from.
|
|
4179
|
+
*
|
|
4180
|
+
* A slice holds `sliceBytes` of records, or every record the read has left to read when that is fewer -
|
|
4181
|
+
* so what it costs depends on how many a read asks for at a time, not on how many it covers. An
|
|
4182
|
+
* iteration asks for a chunk: `iterate({limit: 20_000_000, chunkSize: 1000})` reads a thousand records at
|
|
4183
|
+
* a time however many its window covers, and charging it a full slice would refuse it for 16 MiB it never
|
|
4184
|
+
* allocates. The symbol-usage pass is the exception, because it reads the whole window in slices of its
|
|
4185
|
+
* own before the first chunk is built, so a read that still has that pass ahead of it is charged for it.
|
|
4186
|
+
*
|
|
4187
|
+
* @param {number} windowRows Rows the read covers.
|
|
4188
|
+
* @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
|
|
4189
|
+
* @param {boolean} analysisAhead Whether the symbol-usage pass has still to run.
|
|
4190
|
+
* @return {number} Records read at one time.
|
|
4191
|
+
* @private
|
|
4192
|
+
*/
|
|
4193
|
+
_recordsAtOnce(windowRows, liveRows, analysisAhead) {
|
|
4194
|
+
const chunkRows = liveRows === null ? windowRows : Math.max(1, Math.floor(liveRows.rows / Math.max(1, liveRows.perChunk)));
|
|
4195
|
+
return analysisAhead ? Math.max(windowRows, chunkRows) : chunkRows;
|
|
4196
|
+
}
|
|
4197
|
+
/**
|
|
4198
|
+
* The symbol table's length, as much of it as the file holds: what the header declares, cut short where
|
|
4199
|
+
* the file ends. Known before a byte of the table is read, so everything that can refuse the table is
|
|
4200
|
+
* checked on this, before the table is allocated.
|
|
4201
|
+
*
|
|
4202
|
+
* A file that ends inside its symbol table is measured to where it ends, and the fields whose areas it cut
|
|
4203
|
+
* short are refused as `Symbol data extends beyond buffer` when their metadata is checked - what a
|
|
4204
|
+
* whole-file read has always said of such a file. A window has refused it already, before reading anything.
|
|
4205
|
+
*
|
|
4206
|
+
* @return {number} Bytes.
|
|
4207
|
+
* @private
|
|
4208
|
+
*/
|
|
4209
|
+
_symbolTableLength() {
|
|
4210
|
+
assert4__default.default(
|
|
4211
|
+
this._symbolTableOffset !== null && this._indexTableOffset !== null && this._fileSize !== null,
|
|
4212
|
+
"The QVD file header has not been parsed before its symbol table was measured."
|
|
4213
|
+
);
|
|
4214
|
+
const declared = this._indexTableOffset - this._symbolTableOffset;
|
|
4215
|
+
return Math.max(0, Math.min(declared, this._fileSize - this._symbolTableOffset));
|
|
4216
|
+
}
|
|
4217
|
+
/**
|
|
4218
|
+
* Where each selected field's symbols are, as ranges of the symbol table this read will read.
|
|
4219
|
+
*
|
|
4220
|
+
* A field's `Offset` and `Length` say exactly where its symbols are, so a read of some of a file's fields
|
|
4221
|
+
* has no reason to read the areas of the rest (#122). Qlik writes the areas one after another in field
|
|
4222
|
+
* order, so ranges that touch are merged: a read of every field is one range, and so is a read of fields
|
|
4223
|
+
* that happen to be neighbours. A read of one field of twenty reads that field's area alone.
|
|
4224
|
+
*
|
|
4225
|
+
* Built once per read, from the fields the read selected, and each field's metadata is checked as it is
|
|
4226
|
+
* added - a range is arithmetic on `Offset` and `Length`, and those have to be inside the table first.
|
|
4227
|
+
* `_parseSymbolTable` checks every field of the file, selected or not, before it parses any.
|
|
4228
|
+
*
|
|
4229
|
+
* @return {{ranges: Array<{start: number, end: number, fields: number, buffer: Buffer|null}>,
|
|
4230
|
+
* byField: Map<any, {range: {start: number, end: number, fields: number, buffer: Buffer|null},
|
|
4231
|
+
* start: number, end: number}>}} The ranges, and where in its range each field's area sits.
|
|
4232
|
+
* @private
|
|
4233
|
+
*/
|
|
4234
|
+
_symbolAreaPlan() {
|
|
4235
|
+
if (this._symbolAreas !== null) {
|
|
4236
|
+
return this._symbolAreas;
|
|
4237
|
+
}
|
|
4238
|
+
assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved before their symbols were read.");
|
|
4239
|
+
const tableLength = this._symbolTableLength();
|
|
4240
|
+
const areas = this._selectedFields.map((field) => {
|
|
4241
|
+
validateFieldMetadata(field, tableLength, this._path);
|
|
4242
|
+
const start = headerInteger(field["Offset"]);
|
|
4243
|
+
return { field, start, end: start + headerInteger(field["Length"]) };
|
|
4244
|
+
});
|
|
4245
|
+
const ranges = [];
|
|
4246
|
+
const byField = /* @__PURE__ */ new Map();
|
|
4247
|
+
for (const area of [...areas].sort((a, b) => a.start - b.start)) {
|
|
4248
|
+
const last = ranges.at(-1);
|
|
4249
|
+
const range = last !== void 0 && area.start <= last.end ? last : { start: area.start, end: area.end, fields: 0, buffer: null };
|
|
4250
|
+
if (range !== last) {
|
|
4251
|
+
ranges.push(range);
|
|
4252
|
+
}
|
|
4253
|
+
range.end = Math.max(range.end, area.end);
|
|
4254
|
+
range.fields += 1;
|
|
4255
|
+
byField.set(area.field, { range, start: area.start, end: area.end });
|
|
4256
|
+
}
|
|
4257
|
+
this._symbolAreas = { ranges, byField };
|
|
4258
|
+
return this._symbolAreas;
|
|
4259
|
+
}
|
|
4260
|
+
/**
|
|
4261
|
+
* One field's symbols, as bytes: the range that holds them, read from the open file the first time a field
|
|
4262
|
+
* of that range needs it.
|
|
4263
|
+
*
|
|
4264
|
+
* @param {any} field The field, one this read selected.
|
|
4265
|
+
* @return {Promise<{buffer: Buffer, start: number, end: number, base: number}>} Its area, as a range of
|
|
4266
|
+
* `buffer`, with where that buffer starts in the symbol table - what an error adds back to say where a
|
|
4267
|
+
* damaged symbol is in the file, rather than where it is in the bytes this read happened to read.
|
|
4268
|
+
* @throws {QvdValidationError} If the range is larger than half the heap.
|
|
4269
|
+
* @private
|
|
4270
|
+
*/
|
|
4271
|
+
async _symbolAreaOf(field) {
|
|
4272
|
+
assert4__default.default(
|
|
4273
|
+
this._header && this._symbolTableOffset !== null,
|
|
4274
|
+
"The QVD file header has not been parsed before its symbols were read."
|
|
4275
|
+
);
|
|
4276
|
+
const area = this._symbolAreaPlan().byField.get(field);
|
|
4277
|
+
assert4__default.default(area, "A field this read did not select has no symbol area.");
|
|
4278
|
+
const { range } = area;
|
|
4279
|
+
if (range.buffer === null) {
|
|
4280
|
+
const length = range.end - range.start;
|
|
4281
|
+
validateSymbolTableSize(length, this._path, headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]));
|
|
4282
|
+
const buffer = Buffer.alloc(length);
|
|
4283
|
+
const from = this._symbolTableOffset + range.start;
|
|
4284
|
+
await this._readAt(buffer, 0, length, from, from + length);
|
|
4285
|
+
range.buffer = buffer;
|
|
4286
|
+
}
|
|
4287
|
+
return { buffer: range.buffer, start: area.start - range.start, end: area.end - range.start, base: range.start };
|
|
4288
|
+
}
|
|
4289
|
+
/**
|
|
4290
|
+
* Lets go of a field's symbols once they are parsed, and of the bytes of its range once every field in it
|
|
4291
|
+
* has been.
|
|
4292
|
+
*
|
|
4293
|
+
* Every text is copied out of the bytes as it is decoded, so what the read keeps is the values. A read of
|
|
4294
|
+
* one field of a file whose other fields are large therefore holds that field's bytes and no others.
|
|
4295
|
+
*
|
|
4296
|
+
* @param {any} field The field whose symbols are parsed.
|
|
4297
|
+
* @private
|
|
4298
|
+
*/
|
|
4299
|
+
_releaseSymbolArea(field) {
|
|
4300
|
+
const area = this._symbolAreaPlan().byField.get(field);
|
|
4301
|
+
assert4__default.default(area, "A field this read did not select has no symbol area.");
|
|
4302
|
+
area.range.fields -= 1;
|
|
4303
|
+
if (area.range.fields === 0) {
|
|
4304
|
+
area.range.buffer = null;
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
4307
|
+
/**
|
|
4308
|
+
* Reads records from the open file a slice at a time, and hands each slice to `visit`.
|
|
4309
|
+
*
|
|
4310
|
+
* One buffer of at most `sliceBytes` holds a slice, and is reused for the next one, so a read of any
|
|
4311
|
+
* number of records holds that much of them and no more. Cancellation is checked before each slice.
|
|
4312
|
+
*
|
|
4313
|
+
* @param {number} firstRow The file row of the first record.
|
|
4314
|
+
* @param {number} rowCount How many records.
|
|
4315
|
+
* @param {number} recordSize Bytes per record.
|
|
4316
|
+
* @param {(slice: Buffer, done: number, count: number) => void|Promise<void>} visit Called with each
|
|
4317
|
+
* slice's records, how many records came before it, and how many it holds.
|
|
4318
|
+
* @private
|
|
4319
|
+
*/
|
|
4320
|
+
async _forEachSlice(firstRow, rowCount, recordSize, visit) {
|
|
4321
|
+
if (rowCount === 0) {
|
|
4322
|
+
return;
|
|
4323
|
+
}
|
|
4324
|
+
assert4__default.default(this._indexTableOffset !== null, "The QVD file header has not been parsed before its records were read.");
|
|
4325
|
+
const sliceRows = this._sliceRowsFor(rowCount, recordSize);
|
|
4326
|
+
const slice = Buffer.alloc(sliceRows * recordSize);
|
|
4327
|
+
const requiredBytes = this._indexTableOffset + (firstRow + rowCount) * recordSize;
|
|
4328
|
+
for (let done = 0; done < rowCount; done += sliceRows) {
|
|
4329
|
+
this._throwIfAborted();
|
|
4330
|
+
const count = Math.min(sliceRows, rowCount - done);
|
|
4331
|
+
const records = slice.subarray(0, count * recordSize);
|
|
4332
|
+
await this._readAt(
|
|
4333
|
+
records,
|
|
4334
|
+
0,
|
|
4335
|
+
records.length,
|
|
4336
|
+
this._indexTableOffset + (firstRow + done) * recordSize,
|
|
4337
|
+
requiredBytes
|
|
4338
|
+
);
|
|
4339
|
+
await visit(records, done, count);
|
|
4340
|
+
}
|
|
4341
|
+
}
|
|
3773
4342
|
/**
|
|
3774
4343
|
* Parses the XML header of the QVD file. This method is part of the parsing process
|
|
3775
4344
|
* and should not be called directly.
|
|
3776
4345
|
*/
|
|
3777
4346
|
async _parseHeader() {
|
|
3778
|
-
if (!this.
|
|
4347
|
+
if (!this._headerBuffer) {
|
|
3779
4348
|
throw new exports.QvdCorruptedError(
|
|
3780
4349
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
3781
4350
|
{
|
|
@@ -3786,7 +4355,7 @@ var init_QvdFileReader = __esm({
|
|
|
3786
4355
|
}
|
|
3787
4356
|
const HEADER_DELIMITER = "\r\n\0";
|
|
3788
4357
|
const headerBeginIndex = 0;
|
|
3789
|
-
const headerDelimiterIndex = this.
|
|
4358
|
+
const headerDelimiterIndex = this._headerBuffer.indexOf(HEADER_DELIMITER, headerBeginIndex);
|
|
3790
4359
|
if (headerDelimiterIndex === -1) {
|
|
3791
4360
|
throw new exports.QvdCorruptedError(
|
|
3792
4361
|
"The XML header section does not exist or is not properly delimited from the binary data.",
|
|
@@ -3797,7 +4366,7 @@ var init_QvdFileReader = __esm({
|
|
|
3797
4366
|
);
|
|
3798
4367
|
}
|
|
3799
4368
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3800
|
-
const headerBuffer = this.
|
|
4369
|
+
const headerBuffer = this._headerBuffer.subarray(headerBeginIndex, headerEndIndex);
|
|
3801
4370
|
this._fieldBitMetadataValidated = false;
|
|
3802
4371
|
this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
|
|
3803
4372
|
const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
|
|
@@ -3819,12 +4388,12 @@ var init_QvdFileReader = __esm({
|
|
|
3819
4388
|
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
3820
4389
|
* @param {string} stage Stage name for any error raised here.
|
|
3821
4390
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
3822
|
-
*
|
|
3823
|
-
*
|
|
4391
|
+
* firstRow: number}} The record geometry: the window's `rowsToLoad` records start at file row
|
|
4392
|
+
* `firstRow`, and `_forEachSlice` reads them.
|
|
3824
4393
|
* @private
|
|
3825
4394
|
*/
|
|
3826
4395
|
_planIndexTable(window, stage) {
|
|
3827
|
-
if (!this.
|
|
4396
|
+
if (!this._handle || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
3828
4397
|
throw new exports.QvdCorruptedError(
|
|
3829
4398
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
3830
4399
|
{
|
|
@@ -3839,22 +4408,18 @@ var init_QvdFileReader = __esm({
|
|
|
3839
4408
|
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3840
4409
|
const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
|
|
3841
4410
|
const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
|
|
4411
|
+
assert4__default.default(this._fileSize !== null, "The QVD file has not been measured before its records were planned.");
|
|
3842
4412
|
validateIndexTableMetadata(
|
|
3843
4413
|
recordSize,
|
|
3844
4414
|
totalRows,
|
|
3845
4415
|
indexTableLength,
|
|
3846
4416
|
this._indexTableOffset,
|
|
3847
|
-
this.
|
|
4417
|
+
this._fileSize,
|
|
3848
4418
|
rowsToLoad,
|
|
3849
4419
|
this._path,
|
|
3850
4420
|
this._fileSize,
|
|
3851
4421
|
firstRow,
|
|
3852
|
-
|
|
3853
|
-
);
|
|
3854
|
-
const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
|
|
3855
|
-
const indexBuffer = this._buffer.subarray(
|
|
3856
|
-
this._indexTableOffset + bufferRecordStart,
|
|
3857
|
-
this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
|
|
4422
|
+
0
|
|
3858
4423
|
);
|
|
3859
4424
|
if (!this._fieldBitMetadataValidated) {
|
|
3860
4425
|
for (const field of allFields) {
|
|
@@ -3863,11 +4428,12 @@ var init_QvdFileReader = __esm({
|
|
|
3863
4428
|
validateBitFields(allFields, this._path);
|
|
3864
4429
|
this._fieldBitMetadataValidated = true;
|
|
3865
4430
|
}
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
4431
|
+
const windowEnd = this._indexTableOffset + (firstRow + rowsToLoad) * recordSize;
|
|
4432
|
+
assert4__default.default(
|
|
4433
|
+
rowsToLoad === 0 || recordSize === 0 || windowEnd <= this._fileSize,
|
|
4434
|
+
`The window's records end at byte ${windowEnd} of a file of ${this._fileSize}, but ${rowsToLoad} were validated as present.`
|
|
3869
4435
|
);
|
|
3870
|
-
return { fields, recordSize, totalRows, rowsToLoad,
|
|
4436
|
+
return { fields, recordSize, totalRows, rowsToLoad, firstRow };
|
|
3871
4437
|
}
|
|
3872
4438
|
/**
|
|
3873
4439
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
@@ -3883,48 +4449,58 @@ var init_QvdFileReader = __esm({
|
|
|
3883
4449
|
* @private
|
|
3884
4450
|
*/
|
|
3885
4451
|
async _analyzeIndexTableSymbolUsage(window) {
|
|
3886
|
-
const { fields, recordSize, rowsToLoad,
|
|
4452
|
+
const { fields, recordSize, rowsToLoad, firstRow } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
|
|
3887
4453
|
const symbolUsage = [];
|
|
3888
4454
|
const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
|
|
3889
4455
|
const column = new Int32Array(sliceRows);
|
|
3890
|
-
fields.
|
|
3891
|
-
this._throwIfAborted();
|
|
4456
|
+
const state = fields.map((field) => {
|
|
3892
4457
|
const needed = /* @__PURE__ */ new Set();
|
|
3893
|
-
symbolUsage
|
|
3894
|
-
const bitOffset = headerInteger(field["BitOffset"]);
|
|
3895
|
-
const bitWidth = headerInteger(field["BitWidth"]);
|
|
3896
|
-
const bias = headerInteger(field["Bias"]);
|
|
4458
|
+
symbolUsage.push(needed);
|
|
3897
4459
|
const length = headerInteger(field["Length"]);
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
);
|
|
3911
|
-
for (
|
|
3912
|
-
|
|
3913
|
-
|
|
4460
|
+
return {
|
|
4461
|
+
field,
|
|
4462
|
+
needed,
|
|
4463
|
+
bitOffset: headerInteger(field["BitOffset"]),
|
|
4464
|
+
bitWidth: headerInteger(field["BitWidth"]),
|
|
4465
|
+
bias: headerInteger(field["Bias"]),
|
|
4466
|
+
indexLimit: Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity,
|
|
4467
|
+
counted: false
|
|
4468
|
+
};
|
|
4469
|
+
});
|
|
4470
|
+
await this._forEachSlice(firstRow, rowsToLoad, recordSize, async (records, done, recordCount) => {
|
|
4471
|
+
for (let first = 0; first < recordCount; first += sliceRows) {
|
|
4472
|
+
const count = Math.min(sliceRows, recordCount - first);
|
|
4473
|
+
for (const field of state) {
|
|
4474
|
+
decodeIndexColumn(
|
|
4475
|
+
first === 0 ? records : records.subarray(first * recordSize),
|
|
4476
|
+
recordSize,
|
|
4477
|
+
count,
|
|
4478
|
+
field.bitOffset,
|
|
4479
|
+
field.bitWidth,
|
|
4480
|
+
field.bias,
|
|
4481
|
+
column
|
|
4482
|
+
);
|
|
4483
|
+
for (let row = 0; row < count; row++) {
|
|
4484
|
+
if (column[row] >= 0 && column[row] < field.indexLimit) {
|
|
4485
|
+
field.needed.add(column[row]);
|
|
4486
|
+
}
|
|
3914
4487
|
}
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
4488
|
+
if (!field.counted && field.needed.size > COUNT_SYMBOLS_PAST) {
|
|
4489
|
+
field.counted = true;
|
|
4490
|
+
field.indexLimit = Math.min(field.indexLimit, await this._countFieldSymbols(field.field));
|
|
4491
|
+
for (const index of field.needed) {
|
|
4492
|
+
if (index >= field.indexLimit) {
|
|
4493
|
+
field.needed.delete(index);
|
|
4494
|
+
}
|
|
3922
4495
|
}
|
|
3923
4496
|
}
|
|
3924
4497
|
}
|
|
3925
4498
|
}
|
|
3926
|
-
this._emitProgress("symbol-analysis",
|
|
4499
|
+
this._emitProgress("symbol-analysis", done + recordCount, rowsToLoad);
|
|
3927
4500
|
});
|
|
4501
|
+
if (rowsToLoad === 0) {
|
|
4502
|
+
this._emitProgress("symbol-analysis", 0, 0);
|
|
4503
|
+
}
|
|
3928
4504
|
return symbolUsage;
|
|
3929
4505
|
}
|
|
3930
4506
|
/**
|
|
@@ -3932,28 +4508,27 @@ var init_QvdFileReader = __esm({
|
|
|
3932
4508
|
*
|
|
3933
4509
|
* The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
|
|
3934
4510
|
* `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
|
|
3935
|
-
* validated
|
|
3936
|
-
* same way wherever it is met.
|
|
4511
|
+
* validated before it is read, by `_symbolAreaPlan`, so a damaged `Offset`, `Length` or `NoOfSymbols` is
|
|
4512
|
+
* reported the same way wherever it is met. Its bytes are kept for the parse that follows. The walk checks the count against `NoOfSymbols` as the
|
|
4513
|
+
* parse does, so a field whose count is wrong is refused here, before the pass keeps anything on the
|
|
4514
|
+
* strength of it.
|
|
3937
4515
|
*
|
|
3938
4516
|
* @param {any} field The field's header.
|
|
3939
|
-
* @return {number} Its symbols.
|
|
3940
|
-
* @throws {QvdCorruptedError} If the area is not inside the symbol table,
|
|
4517
|
+
* @return {Promise<number>} Its symbols.
|
|
4518
|
+
* @throws {QvdCorruptedError} If the area is not inside the symbol table, a symbol runs past it, or it
|
|
4519
|
+
* holds a different number of symbols from its `NoOfSymbols`.
|
|
3941
4520
|
* @private
|
|
3942
4521
|
*/
|
|
3943
|
-
_countFieldSymbols(field) {
|
|
3944
|
-
|
|
3945
|
-
this._buffer && this._symbolTableOffset && this._indexTableOffset,
|
|
3946
|
-
"The QVD file has not been read before its symbols were counted."
|
|
3947
|
-
);
|
|
3948
|
-
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3949
|
-
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3950
|
-
const offset = headerInteger(field["Offset"]);
|
|
4522
|
+
async _countFieldSymbols(field) {
|
|
4523
|
+
const area = await this._symbolAreaOf(field);
|
|
3951
4524
|
return countFieldSymbols(
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
4525
|
+
area.buffer,
|
|
4526
|
+
area.start,
|
|
4527
|
+
area.end,
|
|
4528
|
+
headerInteger(field["NoOfSymbols"]),
|
|
3955
4529
|
field["FieldName"],
|
|
3956
|
-
this._path
|
|
4530
|
+
this._path,
|
|
4531
|
+
area.base
|
|
3957
4532
|
);
|
|
3958
4533
|
}
|
|
3959
4534
|
/**
|
|
@@ -3973,7 +4548,7 @@ var init_QvdFileReader = __esm({
|
|
|
3973
4548
|
* that is fewer than the window covers - see `_prepare`.
|
|
3974
4549
|
*/
|
|
3975
4550
|
async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
|
|
3976
|
-
if (!this.
|
|
4551
|
+
if (!this._handle || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
3977
4552
|
throw new exports.QvdCorruptedError(
|
|
3978
4553
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
3979
4554
|
{
|
|
@@ -3984,44 +4559,58 @@ var init_QvdFileReader = __esm({
|
|
|
3984
4559
|
}
|
|
3985
4560
|
const allFields = this._allFields;
|
|
3986
4561
|
const fields = this._selectedFields;
|
|
3987
|
-
const
|
|
3988
|
-
const
|
|
4562
|
+
const symbolTableSize = this._symbolTableLength();
|
|
4563
|
+
const plan = this._symbolAreaPlan();
|
|
4564
|
+
const symbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
|
|
3989
4565
|
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
3990
|
-
|
|
4566
|
+
const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
|
|
4567
|
+
validateSymbolTableSize(symbolBytes, this._path, totalRows);
|
|
3991
4568
|
if (this._headerMatchesFile) {
|
|
3992
4569
|
validateMemoryAvailability(
|
|
3993
|
-
|
|
4570
|
+
symbolBytes,
|
|
3994
4571
|
rowsToLoad,
|
|
3995
4572
|
totalRows,
|
|
3996
4573
|
this._path,
|
|
3997
4574
|
this._memorySafetyFactor,
|
|
3998
4575
|
fields.length,
|
|
3999
4576
|
this._materialisesRows,
|
|
4000
|
-
liveRows
|
|
4577
|
+
liveRows,
|
|
4578
|
+
this._bytesHeld(symbolBytes, rowsToLoad, recordSize, liveRows, false),
|
|
4579
|
+
// `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
|
|
4580
|
+
// run has read the window's records once already - so the read's total is two passes over them.
|
|
4581
|
+
symbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize
|
|
4001
4582
|
);
|
|
4002
4583
|
}
|
|
4003
|
-
warnLargeSymbolTable(
|
|
4584
|
+
warnLargeSymbolTable(symbolBytes, rowsToLoad, totalRows, fields.length, this._materialisesRows);
|
|
4004
4585
|
for (const field of allFields) {
|
|
4005
|
-
validateFieldMetadata(field,
|
|
4586
|
+
validateFieldMetadata(field, symbolTableSize, this._path);
|
|
4006
4587
|
}
|
|
4007
4588
|
validateSymbolAreas(allFields, this._path);
|
|
4008
|
-
|
|
4589
|
+
const symbolTable = [];
|
|
4590
|
+
for (const [position, field] of fields.entries()) {
|
|
4009
4591
|
this._throwIfAborted();
|
|
4010
|
-
const
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
|
|
4592
|
+
const area = await this._symbolAreaOf(field);
|
|
4593
|
+
symbolTable.push(
|
|
4594
|
+
parseFieldSymbols(
|
|
4595
|
+
area.buffer,
|
|
4596
|
+
area.start,
|
|
4597
|
+
area.end,
|
|
4598
|
+
// Checked against the symbols the area holds, which is the one check that sees a terminator
|
|
4599
|
+
// damaged in the middle of it (#124).
|
|
4600
|
+
headerInteger(field["NoOfSymbols"]),
|
|
4601
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
4602
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
4603
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
4604
|
+
field["FieldName"],
|
|
4605
|
+
this._path,
|
|
4606
|
+
void 0,
|
|
4607
|
+
area.base
|
|
4608
|
+
)
|
|
4021
4609
|
);
|
|
4610
|
+
this._releaseSymbolArea(field);
|
|
4022
4611
|
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
4023
|
-
|
|
4024
|
-
|
|
4612
|
+
}
|
|
4613
|
+
this._symbolTable = symbolTable;
|
|
4025
4614
|
}
|
|
4026
4615
|
/**
|
|
4027
4616
|
* Parses the bit stuffed index table of the QVD file. This method is part of the parsing process
|
|
@@ -4048,27 +4637,46 @@ var init_QvdFileReader = __esm({
|
|
|
4048
4637
|
* straight to the caller. Rows outside the window are not decoded, so they are not checked.
|
|
4049
4638
|
*
|
|
4050
4639
|
* @param {QvdRowWindow} window The rows to decode.
|
|
4640
|
+
* @param {number} [progressBase=0] Rows decoded before this call, so that progress over a chunked
|
|
4641
|
+
* iteration counts the whole window rather than restarting at every chunk - what `_buildRows` takes
|
|
4642
|
+
* for the same reason.
|
|
4643
|
+
* @param {number|null} [progressTotal=null] Rows the whole window covers, or null for this call's own.
|
|
4644
|
+
* @param {Array<Int32Array>|null} [into=null] Arrays to decode into, one per selected field and at least
|
|
4645
|
+
* `limit` long, for a caller that decodes chunk after chunk and keeps none of them. Null allocates.
|
|
4051
4646
|
* @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
|
|
4052
4647
|
*/
|
|
4053
|
-
async _parseIndexTable(window) {
|
|
4054
|
-
const { fields, recordSize, rowsToLoad,
|
|
4055
|
-
|
|
4648
|
+
async _parseIndexTable(window, progressBase = 0, progressTotal = null, into = null) {
|
|
4649
|
+
const { fields, recordSize, rowsToLoad, firstRow } = this._planIndexTable(window, "parseIndexTable");
|
|
4650
|
+
const decodedBefore = progressBase;
|
|
4651
|
+
const decodedTotal = progressTotal === null ? rowsToLoad : progressTotal;
|
|
4652
|
+
assert4__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
4056
4653
|
const symbolTable = this._symbolTable;
|
|
4057
|
-
const
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4654
|
+
const decoders = fields.map((field, position) => ({
|
|
4655
|
+
bitOffset: headerInteger(field["BitOffset"]),
|
|
4656
|
+
bitWidth: headerInteger(field["BitWidth"]),
|
|
4657
|
+
bias: headerInteger(field["Bias"]),
|
|
4658
|
+
symbolCount: symbolTable[position].numbers.length,
|
|
4659
|
+
name: field["FieldName"]
|
|
4660
|
+
}));
|
|
4661
|
+
const columns = into === null ? fields.map(() => new Int32Array(rowsToLoad)) : into.map((codes) => codes.subarray(0, rowsToLoad));
|
|
4662
|
+
await this._forEachSlice(firstRow, rowsToLoad, recordSize, (records, done, count) => {
|
|
4663
|
+
decoders.forEach((decoder, position) => {
|
|
4664
|
+
decodeIndexColumn(
|
|
4665
|
+
records,
|
|
4666
|
+
recordSize,
|
|
4667
|
+
count,
|
|
4668
|
+
decoder.bitOffset,
|
|
4669
|
+
decoder.bitWidth,
|
|
4670
|
+
decoder.bias,
|
|
4671
|
+
columns[position].subarray(done, done + count),
|
|
4672
|
+
{ symbolCount: decoder.symbolCount, field: decoder.name, file: this._path, firstRow: firstRow + done }
|
|
4673
|
+
);
|
|
4674
|
+
});
|
|
4675
|
+
this._emitProgress("index-table", decodedBefore + done + count, decodedTotal);
|
|
4071
4676
|
});
|
|
4677
|
+
if (rowsToLoad === 0) {
|
|
4678
|
+
this._emitProgress("index-table", decodedBefore, decodedTotal);
|
|
4679
|
+
}
|
|
4072
4680
|
this._indexColumns = columns;
|
|
4073
4681
|
this._rowsDecoded = rowsToLoad;
|
|
4074
4682
|
}
|
|
@@ -4087,31 +4695,122 @@ var init_QvdFileReader = __esm({
|
|
|
4087
4695
|
* @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
|
|
4088
4696
|
*/
|
|
4089
4697
|
async loadMetadata() {
|
|
4090
|
-
await this.
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4698
|
+
return await this._closingAfter(async () => {
|
|
4699
|
+
await this._readData({ offset: 0, limit: null }, true);
|
|
4700
|
+
this._emitProgress("header", 0, 1);
|
|
4701
|
+
await this._parseHeader();
|
|
4702
|
+
this._emitProgress("header", 1, 1);
|
|
4703
|
+
this._throwIfAborted();
|
|
4704
|
+
assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
4705
|
+
const header = this._header["QvdTableHeader"];
|
|
4706
|
+
const columns = this._allFields.map((field) => field["FieldName"]);
|
|
4707
|
+
const rowCount = headerInteger(header["NoOfRecords"]);
|
|
4708
|
+
validateRecordCount(rowCount, this._path, "readMetadata");
|
|
4709
|
+
const shape = new exports.QvdDataFrame([], columns, header, {
|
|
4710
|
+
symbolTableBytes: headerInteger(header["Offset"]),
|
|
4711
|
+
totalRows: rowCount,
|
|
4712
|
+
rowsLoaded: 0,
|
|
4713
|
+
symbolFiltering: false,
|
|
4714
|
+
symbolsKept: null
|
|
4715
|
+
});
|
|
4716
|
+
return {
|
|
4717
|
+
columns,
|
|
4718
|
+
rowCount,
|
|
4719
|
+
columnCount: columns.length,
|
|
4720
|
+
fields: columns.map((name) => shape.getFieldMetadata(name)),
|
|
4721
|
+
fileMetadata: shape.fileMetadata,
|
|
4722
|
+
metadata: header
|
|
4723
|
+
};
|
|
4724
|
+
});
|
|
4725
|
+
}
|
|
4726
|
+
/**
|
|
4727
|
+
* What a read of this file would cost, and whether it fits, without reading it.
|
|
4728
|
+
*
|
|
4729
|
+
* Reads the header and the file's size and nothing else, at the constant cost of `loadMetadata()`,
|
|
4730
|
+
* then asks the same question a read asks before it allocates anything - through the same function,
|
|
4731
|
+
* from the same numbers. That is the whole point: an answer computed a second way would be a second
|
|
4732
|
+
* opinion, and a read this approves would still be refused.
|
|
4733
|
+
*
|
|
4734
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [rawWindow]
|
|
4735
|
+
* The rows the read would cover, spelled any of the ways a read accepts.
|
|
4736
|
+
* @param {{chunkSize?: number|null}} [options] `chunkSize` when the read would be an `iterate()`,
|
|
4737
|
+
* which holds two chunks of rows rather than the window.
|
|
4738
|
+
* @return {Promise<any>} The answer - see `checkMemory`.
|
|
4739
|
+
*/
|
|
4740
|
+
async checkRead(rawWindow, { chunkSize = null } = {}) {
|
|
4741
|
+
const window = normaliseWindow(rawWindow, this._path);
|
|
4742
|
+
return await this._closingAfter(async () => {
|
|
4743
|
+
await this._readData({ offset: 0, limit: null }, true);
|
|
4744
|
+
this._emitProgress("header", 0, 1);
|
|
4745
|
+
await this._parseHeader();
|
|
4746
|
+
this._emitProgress("header", 1, 1);
|
|
4747
|
+
this._throwIfAborted();
|
|
4748
|
+
assert4__default.default(
|
|
4749
|
+
this._header && this._selectedFields && this._allFields && this._symbolTableOffset !== null,
|
|
4750
|
+
"The QVD file header has not been parsed."
|
|
4751
|
+
);
|
|
4752
|
+
const header = this._header["QvdTableHeader"];
|
|
4753
|
+
const totalRows = headerInteger(header["NoOfRecords"]);
|
|
4754
|
+
const recordSize = headerInteger(header["RecordByteSize"]);
|
|
4755
|
+
const symbolTableLength = headerInteger(header["Offset"]);
|
|
4756
|
+
const selected = this._selectedFields;
|
|
4757
|
+
validateRecordSize(recordSize, this._path, "checkRead");
|
|
4758
|
+
validateRecordCount(totalRows, this._path, "checkRead");
|
|
4759
|
+
if (!this._headerMatchesFile) {
|
|
4760
|
+
throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
|
|
4761
|
+
file: this._path,
|
|
4762
|
+
fileSize: this._fileSize,
|
|
4763
|
+
requiredBytes: this._symbolTableOffset + symbolTableLength + totalRows * recordSize,
|
|
4764
|
+
stage: "checkRead"
|
|
4765
|
+
});
|
|
4766
|
+
}
|
|
4767
|
+
const tableLength = this._symbolTableLength();
|
|
4768
|
+
for (const field of this._allFields) {
|
|
4769
|
+
validateFieldMetadata(field, tableLength, this._path);
|
|
4770
|
+
validateFieldBitMetadata(field, recordSize, this._path);
|
|
4771
|
+
}
|
|
4772
|
+
validateSymbolAreas(this._allFields, this._path);
|
|
4773
|
+
const resolved = resolveWindow(window, totalRows);
|
|
4774
|
+
const windowRows = resolved.limit;
|
|
4775
|
+
const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
|
|
4776
|
+
const analysisAhead = this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
|
|
4777
|
+
const measured = getMemoryBudget();
|
|
4778
|
+
const ask = /* @__PURE__ */ __name((fields, rows) => {
|
|
4779
|
+
const bytes = symbolBytesOf(fields, symbolTableLength);
|
|
4780
|
+
return checkMemory({
|
|
4781
|
+
measured,
|
|
4782
|
+
symbolTableSize: bytes,
|
|
4783
|
+
maxRows: rows,
|
|
4784
|
+
totalRows,
|
|
4785
|
+
safetyFactor: this._memorySafetyFactor,
|
|
4786
|
+
columnCount: fields.length,
|
|
4787
|
+
materialisesRows: this._materialisesRows,
|
|
4788
|
+
live: liveRows,
|
|
4789
|
+
bytesHeld: this._bytesHeld(bytes, rows, recordSize, liveRows, analysisAhead),
|
|
4790
|
+
// What it reads from the file, which is not what it holds: the symbol areas, and every record
|
|
4791
|
+
// the window covers, read a slice at a time and not kept - twice over where the symbol-usage
|
|
4792
|
+
// pass will run, since it reads them before the decode reads them again.
|
|
4793
|
+
readBytes: bytes + readPasses(analysisAhead) * windowRows * recordSize
|
|
4794
|
+
});
|
|
4795
|
+
}, "ask");
|
|
4796
|
+
const answer = ask(selected, windowRows);
|
|
4797
|
+
if (!answer.fits && selected.length > 1) {
|
|
4798
|
+
const bySize = [...selected].sort(
|
|
4799
|
+
(a, b) => headerInteger(a["Length"]) - headerInteger(b["Length"])
|
|
4800
|
+
);
|
|
4801
|
+
for (let take = selected.length - 1; take >= 1; take -= 1) {
|
|
4802
|
+
const fewer = bySize.slice(0, take);
|
|
4803
|
+
if (ask(fewer, windowRows).fits) {
|
|
4804
|
+
answer.suggestions.push({
|
|
4805
|
+
option: "fields",
|
|
4806
|
+
value: fewer.map((field) => field["FieldName"])
|
|
4807
|
+
});
|
|
4808
|
+
break;
|
|
4809
|
+
}
|
|
4810
|
+
}
|
|
4811
|
+
}
|
|
4812
|
+
return answer;
|
|
4106
4813
|
});
|
|
4107
|
-
return {
|
|
4108
|
-
columns,
|
|
4109
|
-
rowCount,
|
|
4110
|
-
columnCount: columns.length,
|
|
4111
|
-
fields: columns.map((name) => shape.getFieldMetadata(name)),
|
|
4112
|
-
fileMetadata: shape.fileMetadata,
|
|
4113
|
-
metadata: header
|
|
4114
|
-
};
|
|
4115
4814
|
}
|
|
4116
4815
|
/**
|
|
4117
4816
|
* Loads the QVD file into memory and parses it.
|
|
@@ -4126,19 +4825,21 @@ var init_QvdFileReader = __esm({
|
|
|
4126
4825
|
*/
|
|
4127
4826
|
async load(window = null) {
|
|
4128
4827
|
const rows = normaliseWindow(window, this._path);
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4828
|
+
return await this._closingAfter(async () => {
|
|
4829
|
+
const prepared = await this._prepare(rows);
|
|
4830
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
4831
|
+
const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
|
|
4832
|
+
return new exports.QvdDataFrame(
|
|
4833
|
+
data,
|
|
4834
|
+
prepared.columns,
|
|
4835
|
+
prepared.metadata,
|
|
4836
|
+
{
|
|
4837
|
+
...prepared.loadStats,
|
|
4838
|
+
rowsLoaded: data.length
|
|
4839
|
+
},
|
|
4840
|
+
prepared.storedSymbols
|
|
4841
|
+
);
|
|
4842
|
+
});
|
|
4142
4843
|
}
|
|
4143
4844
|
/**
|
|
4144
4845
|
* Reads the file as columns, without ever materialising rows.
|
|
@@ -4157,19 +4858,21 @@ var init_QvdFileReader = __esm({
|
|
|
4157
4858
|
*/
|
|
4158
4859
|
async loadColumnar(window = null) {
|
|
4159
4860
|
const rows = normaliseWindow(window, this._path);
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4861
|
+
return await this._closingAfter(async () => {
|
|
4862
|
+
const prepared = await this._prepare(rows, null, true);
|
|
4863
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
4864
|
+
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
4865
|
+
assert4__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
4866
|
+
return new QvdColumnTable2({
|
|
4867
|
+
columns: prepared.columns,
|
|
4868
|
+
codesByField: this._indexColumns,
|
|
4869
|
+
symbolsByField: prepared.resolvedByField,
|
|
4870
|
+
halvesByField: prepared.halvesByField,
|
|
4871
|
+
rowCount: this._rowsDecoded,
|
|
4872
|
+
metadata: prepared.metadata,
|
|
4873
|
+
storedSymbols: prepared.storedSymbols,
|
|
4874
|
+
loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
|
|
4875
|
+
});
|
|
4173
4876
|
});
|
|
4174
4877
|
}
|
|
4175
4878
|
/**
|
|
@@ -4198,37 +4901,41 @@ var init_QvdFileReader = __esm({
|
|
|
4198
4901
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
|
|
4199
4902
|
*/
|
|
4200
4903
|
async *iterateRows(window, chunkSize) {
|
|
4201
|
-
|
|
4202
|
-
throw new exports.QvdValidationError("chunkSize must be a positive integer", {
|
|
4203
|
-
provided: chunkSize,
|
|
4204
|
-
type: typeof chunkSize,
|
|
4205
|
-
file: this._path
|
|
4206
|
-
});
|
|
4207
|
-
}
|
|
4904
|
+
requireChunkSize(chunkSize, this._path);
|
|
4208
4905
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
4209
4906
|
const rows = normaliseWindow(window, this._path);
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
const
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
prepared.
|
|
4224
|
-
prepared.
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4907
|
+
this._startRead();
|
|
4908
|
+
let failing = false;
|
|
4909
|
+
try {
|
|
4910
|
+
const prepared = await this._prepare(rows, liveRows);
|
|
4911
|
+
if (prepared.rowsAvailable === 0) {
|
|
4912
|
+
this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
|
|
4913
|
+
return;
|
|
4914
|
+
}
|
|
4915
|
+
const codes = prepared.columns.map(() => new Int32Array(Math.min(chunkSize, prepared.rowsAvailable)));
|
|
4916
|
+
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
4917
|
+
this._throwIfAborted();
|
|
4918
|
+
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
4919
|
+
const offset = prepared.offset + done;
|
|
4920
|
+
await this._parseIndexTable({ offset, limit: count }, done, prepared.rowsAvailable, codes);
|
|
4921
|
+
const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
|
|
4922
|
+
yield new exports.QvdDataFrame(
|
|
4923
|
+
data,
|
|
4924
|
+
prepared.columns,
|
|
4925
|
+
prepared.metadata,
|
|
4926
|
+
{
|
|
4927
|
+
...prepared.loadStats,
|
|
4928
|
+
offset,
|
|
4929
|
+
rowsLoaded: data.length
|
|
4930
|
+
},
|
|
4931
|
+
prepared.storedSymbols
|
|
4932
|
+
);
|
|
4933
|
+
}
|
|
4934
|
+
} catch (error) {
|
|
4935
|
+
failing = true;
|
|
4936
|
+
throw error;
|
|
4937
|
+
} finally {
|
|
4938
|
+
await this._endRead(failing);
|
|
4232
4939
|
}
|
|
4233
4940
|
}
|
|
4234
4941
|
/**
|
|
@@ -4262,23 +4969,21 @@ var init_QvdFileReader = __esm({
|
|
|
4262
4969
|
await this._parseHeader();
|
|
4263
4970
|
this._emitProgress("header", 1, 1);
|
|
4264
4971
|
this._throwIfAborted();
|
|
4265
|
-
|
|
4972
|
+
assert4__default.default(this._header, "The QVD file header has not been parsed.");
|
|
4266
4973
|
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
4267
4974
|
const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
|
|
4268
4975
|
const resolved = resolveWindow(window, totalRows);
|
|
4269
4976
|
const rowsAvailable = resolved.limit;
|
|
4270
4977
|
let symbolsToKeep = null;
|
|
4271
4978
|
let symbolsKept = null;
|
|
4272
|
-
if (window
|
|
4273
|
-
|
|
4274
|
-
|
|
4275
|
-
symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
|
|
4276
|
-
}
|
|
4979
|
+
if (this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) {
|
|
4980
|
+
symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
|
|
4981
|
+
symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
|
|
4277
4982
|
}
|
|
4278
4983
|
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
4279
|
-
|
|
4984
|
+
assert4__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
4280
4985
|
this._throwIfAborted();
|
|
4281
|
-
|
|
4986
|
+
assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
4282
4987
|
const resolvedByField = [];
|
|
4283
4988
|
const halvesByField = [];
|
|
4284
4989
|
const entries = [];
|
|
@@ -4339,7 +5044,7 @@ var init_QvdFileReader = __esm({
|
|
|
4339
5044
|
* @private
|
|
4340
5045
|
*/
|
|
4341
5046
|
_buildRows(resolvedByField, progressBase, progressTotal) {
|
|
4342
|
-
|
|
5047
|
+
assert4__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
4343
5048
|
const indexColumns = this._indexColumns;
|
|
4344
5049
|
const fieldCount = indexColumns.length;
|
|
4345
5050
|
const rowCount = this._rowsDecoded;
|
|
@@ -5028,12 +5733,16 @@ var init_QvdDataFrame = __esm({
|
|
|
5028
5733
|
/**
|
|
5029
5734
|
* Reads a QVD file in chunks, as an async generator of data frames.
|
|
5030
5735
|
*
|
|
5031
|
-
* The file is opened
|
|
5032
|
-
*
|
|
5033
|
-
*
|
|
5034
|
-
*
|
|
5035
|
-
*
|
|
5036
|
-
*
|
|
5736
|
+
* The file is opened once and its symbol table parsed once. Each chunk's records are read from the
|
|
5737
|
+
* file when that chunk is built, so what this holds is the symbol table and two chunks of rows,
|
|
5738
|
+
* whatever the size of the file: a 20 GB QVD iterates in the memory its symbol table needs. That
|
|
5739
|
+
* table is still parsed in full whatever the chunk size, because a stored index in the last chunk
|
|
5740
|
+
* can address the first symbol. On a high-cardinality file that table is the bulk of the cost, and
|
|
5741
|
+
* `readMetadata` is the only read that avoids it.
|
|
5742
|
+
*
|
|
5743
|
+
* The file stays open until the iteration ends. Running it to the end closes it, and so do
|
|
5744
|
+
* `break` or a throw inside `for await` and a call to `return()` on the iterator; an iterator
|
|
5745
|
+
* abandoned part-way without any of those holds the file until it is garbage-collected.
|
|
5037
5746
|
*
|
|
5038
5747
|
* ```js
|
|
5039
5748
|
* for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
|
|
@@ -5107,6 +5816,68 @@ var init_QvdDataFrame = __esm({
|
|
|
5107
5816
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
5108
5817
|
return await new QvdFileReader2(path5, metadataOptionsFrom(options)).loadMetadata();
|
|
5109
5818
|
}
|
|
5819
|
+
/**
|
|
5820
|
+
* Answers what a read would cost, and whether it fits, without doing it.
|
|
5821
|
+
*
|
|
5822
|
+
* Takes the options `fromQvd()` takes, plus `as` and `chunkSize` to say which read is being asked
|
|
5823
|
+
* about. Reads the header and the file's size and nothing else, at a cost that does not grow with
|
|
5824
|
+
* the file.
|
|
5825
|
+
*
|
|
5826
|
+
* The answer comes from the same function a read consults before it allocates anything, from the
|
|
5827
|
+
* same numbers, so **a read this approves is not refused later for memory** - and every suggestion
|
|
5828
|
+
* it carries has been read back through the check, so following one gives a read that fits.
|
|
5829
|
+
*
|
|
5830
|
+
* It answers about resources, so it answers only for a header it can trust. A header whose numbers
|
|
5831
|
+
* are not usable, or that claims more than the file holds, is refused as a `QvdCorruptedError` rather
|
|
5832
|
+
* than answered: sizing a read from numbers the file contradicts produced a memory verdict about a
|
|
5833
|
+
* file whose real problem was structural, and it was wrong in both directions - approving a read the
|
|
5834
|
+
* library then refused, and refusing another with advice that was refused too.
|
|
5835
|
+
*
|
|
5836
|
+
* That covers everything a reader can tell from the header: a field area past the end of the symbol
|
|
5837
|
+
* table, two fields claiming one area, a `Bias` that is neither 0 nor -2, a `BitWidth` past 31. Damage
|
|
5838
|
+
* that is not in the header - a value or an index the file has spoiled - is still found only by
|
|
5839
|
+
* reading, and still refused as a `QvdCorruptedError` after this has said the read fits.
|
|
5840
|
+
*
|
|
5841
|
+
* ```js
|
|
5842
|
+
* const answer = await QvdDataFrame.checkRead('huge.qvd', {as: 'columns', fields: ['Amount']});
|
|
5843
|
+
*
|
|
5844
|
+
* if (!answer.fits) {
|
|
5845
|
+
* console.log(answer.reason); // 'memory'
|
|
5846
|
+
* console.log(answer.suggestions); // [{option: 'limit', value: 1250000}, ...]
|
|
5847
|
+
* }
|
|
5848
|
+
* ```
|
|
5849
|
+
*
|
|
5850
|
+
* @param {string} path The QVD file.
|
|
5851
|
+
* @param {object} [options] What `fromQvd()` takes, plus the two below.
|
|
5852
|
+
* @param {'rows'|'columns'} [options.as='rows'] Which read is being asked about: `rows` builds row
|
|
5853
|
+
* arrays and `columns` does not, which is most of what a read costs.
|
|
5854
|
+
* @param {number|null} [options.chunkSize=null] The chunk an `iterate()` would use, which holds two
|
|
5855
|
+
* chunks of rows rather than the whole window.
|
|
5856
|
+
* @return {Promise<any>} The answer: `fits`, `reason` when it does not, `estimate`, `budget`,
|
|
5857
|
+
* `exact` and `suggestions`.
|
|
5858
|
+
* @throws {QvdValidationError} If an option's value is not valid, with `context.reason` of `option`.
|
|
5859
|
+
* @throws {QvdCorruptedError} If the header cannot be read, its numbers are not usable, or it claims
|
|
5860
|
+
* more than the file holds. The read refuses such a file too, though it may name the fault
|
|
5861
|
+
* differently - it gets there by planning the index table, where this gets there from the size.
|
|
5862
|
+
*/
|
|
5863
|
+
static async checkRead(path5, options = {}) {
|
|
5864
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
5865
|
+
const { as = "rows", chunkSize = null } = options;
|
|
5866
|
+
if (as !== "rows" && as !== "columns") {
|
|
5867
|
+
throw new exports.QvdValidationError("as must be 'rows' or 'columns'", {
|
|
5868
|
+
provided: as,
|
|
5869
|
+
reason: "option",
|
|
5870
|
+
option: "as",
|
|
5871
|
+
value: as,
|
|
5872
|
+
file: path5
|
|
5873
|
+
});
|
|
5874
|
+
}
|
|
5875
|
+
if (chunkSize !== null) {
|
|
5876
|
+
requireChunkSize(chunkSize, path5);
|
|
5877
|
+
}
|
|
5878
|
+
const reader = new QvdFileReader2(path5, { ...readerOptionsFrom(options), materialisesRows: as === "rows" });
|
|
5879
|
+
return await reader.checkRead(windowFrom(options), { chunkSize });
|
|
5880
|
+
}
|
|
5110
5881
|
/**
|
|
5111
5882
|
* Constructs a data frame from a dictionary.
|
|
5112
5883
|
*
|