qvdjs 2.0.5 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  var fs2 = require('fs');
4
4
  var path2 = require('path');
5
- var assert3 = require('assert');
5
+ var assert4 = require('assert');
6
6
  var crypto2 = require('crypto');
7
7
  var promises = require('timers/promises');
8
8
  var xml = require('xml2js');
@@ -14,7 +14,7 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
14
14
 
15
15
  var fs2__default = /*#__PURE__*/_interopDefault(fs2);
16
16
  var path2__default = /*#__PURE__*/_interopDefault(path2);
17
- var assert3__default = /*#__PURE__*/_interopDefault(assert3);
17
+ var assert4__default = /*#__PURE__*/_interopDefault(assert4);
18
18
  var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
19
19
  var xml__default = /*#__PURE__*/_interopDefault(xml);
20
20
  var os__default = /*#__PURE__*/_interopDefault(os);
@@ -453,7 +453,9 @@ var init_optionTypes = __esm({
453
453
  function requireRowCount(value, name, filePath) {
454
454
  if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
455
455
  throw new exports.QvdValidationError(`${name} must be a non-negative integer`, {
456
+ reason: "option",
456
457
  option: name,
458
+ value,
457
459
  provided: value,
458
460
  type: typeof value,
459
461
  file: filePath
@@ -470,6 +472,9 @@ function normaliseWindow(window, filePath) {
470
472
  }
471
473
  if (typeof window !== "object" || Array.isArray(window)) {
472
474
  throw new exports.QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
475
+ reason: "option",
476
+ option: "limit",
477
+ value: window,
473
478
  provided: window,
474
479
  type: typeof window,
475
480
  file: filePath
@@ -480,6 +485,9 @@ function normaliseWindow(window, filePath) {
480
485
  const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
481
486
  if (limitGiven && maxRowsGiven) {
482
487
  throw new exports.QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
488
+ reason: "option",
489
+ option: "limit",
490
+ value: limit,
483
491
  maxRows,
484
492
  limit,
485
493
  file: filePath
@@ -490,6 +498,19 @@ function normaliseWindow(window, filePath) {
490
498
  limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
491
499
  };
492
500
  }
501
+ function requireChunkSize(chunkSize, filePath) {
502
+ if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
503
+ throw new exports.QvdValidationError("chunkSize must be a positive integer", {
504
+ reason: "option",
505
+ option: "chunkSize",
506
+ value: chunkSize,
507
+ provided: chunkSize,
508
+ type: typeof chunkSize,
509
+ file: filePath
510
+ });
511
+ }
512
+ return chunkSize;
513
+ }
493
514
  function resolveWindow(window, totalRows) {
494
515
  const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
495
516
  const offset = Math.min(window.offset, rows);
@@ -504,6 +525,9 @@ function selectFields(fields, requested, filePath) {
504
525
  }
505
526
  if (!Array.isArray(requested)) {
506
527
  throw new exports.QvdValidationError("fields must be an array of field names", {
528
+ reason: "option",
529
+ option: "fields",
530
+ value: requested,
507
531
  provided: requested,
508
532
  type: typeof requested,
509
533
  file: filePath
@@ -512,6 +536,9 @@ function selectFields(fields, requested, filePath) {
512
536
  const available = fields.map((field) => field["FieldName"]);
513
537
  if (requested.length === 0) {
514
538
  throw new exports.QvdValidationError("fields must name at least one field", {
539
+ reason: "option",
540
+ option: "fields",
541
+ value: requested,
515
542
  availableColumns: available,
516
543
  file: filePath
517
544
  });
@@ -520,6 +547,9 @@ function selectFields(fields, requested, filePath) {
520
547
  return requested.map((name) => {
521
548
  if (typeof name !== "string") {
522
549
  throw new exports.QvdValidationError("Field names must be strings", {
550
+ reason: "option",
551
+ option: "fields",
552
+ value: name,
523
553
  provided: name,
524
554
  type: typeof name,
525
555
  availableColumns: available,
@@ -528,6 +558,9 @@ function selectFields(fields, requested, filePath) {
528
558
  }
529
559
  if (seen.has(name)) {
530
560
  throw new exports.QvdValidationError(`Field '${name}' is listed twice`, {
561
+ reason: "option",
562
+ option: "fields",
563
+ value: name,
531
564
  column: name,
532
565
  fields: requested,
533
566
  file: filePath
@@ -537,6 +570,9 @@ function selectFields(fields, requested, filePath) {
537
570
  const index = available.indexOf(name);
538
571
  if (index === -1) {
539
572
  throw new exports.QvdValidationError(`Column '${name}' does not exist`, {
573
+ reason: "option",
574
+ option: "fields",
575
+ value: name,
540
576
  column: name,
541
577
  availableColumns: available,
542
578
  file: filePath
@@ -551,7 +587,9 @@ function normaliseDuals(value, filePath) {
551
587
  }
552
588
  if (!DUAL_MODES.includes(value)) {
553
589
  throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
590
+ reason: "option",
554
591
  option: "duals",
592
+ value,
555
593
  provided: value,
556
594
  file: filePath
557
595
  });
@@ -590,6 +628,7 @@ var init_readOptions = __esm({
590
628
  init_optionTypes();
591
629
  __name(requireRowCount, "requireRowCount");
592
630
  __name(normaliseWindow, "normaliseWindow");
631
+ __name(requireChunkSize, "requireChunkSize");
593
632
  __name(resolveWindow, "resolveWindow");
594
633
  __name(selectFields, "selectFields");
595
634
  DUAL_MODES = Object.freeze(["number", "text", "both"]);
@@ -971,7 +1010,7 @@ function changedAfterCheck(checked, change) {
971
1010
  });
972
1011
  }
973
1012
  async function openChecked(checked, purpose, failed, { nofollow = NOFOLLOW } = {}) {
974
- assert3__default.default(purpose === "read" || checked.stats !== null, "A rewrite in place is of a file that exists.");
1013
+ assert4__default.default(purpose === "read" || checked.stats !== null, "A rewrite in place is of a file that exists.");
975
1014
  const noFollow = checked.onDisk ? nofollow : 0;
976
1015
  const flags = (purpose === "rewrite" ? O_WRONLY : O_RDONLY) | noFollow;
977
1016
  let handle;
@@ -1547,9 +1586,9 @@ var init_QvdFileWriter = __esm({
1547
1586
  * Writes the data to the QVD file.
1548
1587
  */
1549
1588
  async _writeData() {
1550
- assert3__default.default(this._header, "The QVD file header has not been parsed.");
1551
- assert3__default.default(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
1552
- assert3__default.default(this._indexBuffer, "The QVD file index table has not been parsed.");
1589
+ assert4__default.default(this._header, "The QVD file header has not been parsed.");
1590
+ assert4__default.default(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
1591
+ assert4__default.default(this._indexBuffer, "The QVD file index table has not been parsed.");
1553
1592
  this._emitProgress("write", 0, 1);
1554
1593
  const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
1555
1594
  const failed = rethrowAsIoError(this._path, "write");
@@ -1997,7 +2036,7 @@ var init_QvdFileWriter = __esm({
1997
2036
  const key = keys[slot];
1998
2037
  offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
1999
2038
  }
2000
- assert3__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
2039
+ assert4__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
2001
2040
  columnBuffers.push(columnBuffer);
2002
2041
  this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
2003
2042
  this._symbolCounts?.push(keys.length);
@@ -2036,9 +2075,9 @@ var init_QvdFileWriter = __esm({
2036
2075
  * @private
2037
2076
  */
2038
2077
  _buildIndexTable() {
2039
- assert3__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
2040
- assert3__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
2041
- assert3__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
2078
+ assert4__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
2079
+ assert4__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
2080
+ assert4__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
2042
2081
  this._indexTableMetadata = [];
2043
2082
  const columns = this._df.columns;
2044
2083
  const data = this._df.data;
@@ -2218,20 +2257,57 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
2218
2257
  }
2219
2258
  return low;
2220
2259
  }
2221
- function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
2222
- if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
2223
- throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
2224
- }
2225
- if (safetyFactor === 0) {
2260
+ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null) {
2261
+ const answer = checkMemory({
2262
+ symbolTableSize,
2263
+ maxRows,
2264
+ totalRows,
2265
+ safetyFactor,
2266
+ columnCount,
2267
+ materialisesRows,
2268
+ live,
2269
+ bytesHeld,
2270
+ readBytes
2271
+ });
2272
+ if (answer.fits) {
2226
2273
  return;
2227
2274
  }
2228
- const budget = getMemoryBudget();
2275
+ const { message, context } = answer.refusal;
2276
+ throw new exports.QvdValidationError(message, {
2277
+ file: filePath,
2278
+ ...context,
2279
+ reason: "memory",
2280
+ check: answerOf(answer)
2281
+ });
2282
+ }
2283
+ function checkMemory({
2284
+ symbolTableSize,
2285
+ maxRows,
2286
+ totalRows,
2287
+ safetyFactor = 0.8,
2288
+ columnCount = 0,
2289
+ materialisesRows = true,
2290
+ live = null,
2291
+ bytesHeld = null,
2292
+ readBytes = null,
2293
+ measured = null
2294
+ }) {
2295
+ if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
2296
+ throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
2297
+ safetyFactor,
2298
+ reason: "option",
2299
+ option: "memorySafetyFactor",
2300
+ value: safetyFactor
2301
+ });
2302
+ }
2303
+ const budget = measured ?? getMemoryBudget();
2229
2304
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
2230
2305
  const rowsLive = live === null ? null : live.rows;
2231
2306
  const liveRowsPerChunk = live === null ? 1 : live.perChunk;
2232
2307
  const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
2233
2308
  const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
2234
- const externalMemory = estimateExternalMemory(liveRows, columnCount);
2309
+ const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
2310
+ const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
2235
2311
  const bounded = budget.candidates.map((candidate) => {
2236
2312
  const heapOnly = candidate.source === "V8 heap limit";
2237
2313
  return {
@@ -2242,25 +2318,45 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
2242
2318
  bounds: heapOnly ? "the V8 heap" : "the whole process"
2243
2319
  };
2244
2320
  });
2245
- const exceeded = bounded.filter((candidate) => candidate.needs > candidate.allowed);
2246
- const binding = exceeded.reduce(
2247
- (worst, candidate) => candidate.needs / candidate.allowed > worst.needs / worst.allowed ? candidate : worst,
2248
- exceeded[0]
2321
+ if (safetyFactor === 0) {
2322
+ const lowest = budget.candidates.reduce((least, candidate) => candidate.bytes < least.bytes ? candidate : least);
2323
+ return {
2324
+ fits: true,
2325
+ estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
2326
+ // The ceilings are still there and still named - what is missing is any measurement against them.
2327
+ // An earlier version reported `bound: 'none'` here, a third value in a two-value vocabulary that a
2328
+ // caller switching on the documented two would fall straight through.
2329
+ budget: {
2330
+ ...budgetOf(budget, { ...lowest, heapOnly: lowest.source === "V8 heap limit" }, 0),
2331
+ allowedBytes: Infinity
2332
+ },
2333
+ exact: symbolTableSize === 0,
2334
+ suggestions: []
2335
+ };
2336
+ }
2337
+ const tightest = bounded.reduce(
2338
+ (worst, candidate) => candidate.needs / candidate.allowed > worst.needs / worst.allowed ? candidate : worst
2249
2339
  );
2340
+ const binding = tightest.needs > tightest.allowed ? tightest : null;
2250
2341
  const heapLimit = getHeapLimit();
2251
2342
  const availableMemory = binding ? binding.bytes : budget.bytes;
2252
2343
  const estimatedMemory = binding ? binding.needs : heapMemory;
2253
2344
  const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
2254
2345
  if (binding) {
2255
2346
  const includeExternal = !binding.heapOnly;
2256
- const recommendedMaxRows = recommendedRowsFor(
2257
- maxAllowedMemory,
2347
+ const rowsHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForRows(rows) : 0), "rowsHeldBudget");
2348
+ const fitting = /* @__PURE__ */ __name((rows) => recommendedRowsFor(
2349
+ rowsHeldBudget(rows),
2258
2350
  symbolTableSize,
2259
2351
  totalRows,
2260
2352
  columnCount,
2261
2353
  materialisesRows,
2262
2354
  includeExternal
2263
- );
2355
+ ), "fitting");
2356
+ const firstGuess = fitting(liveRows);
2357
+ const over = fitting(firstGuess);
2358
+ const under = fitting(over);
2359
+ const recommendedMaxRows = Math.max(firstGuess, under);
2264
2360
  const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
2265
2361
  const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
2266
2362
  const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
@@ -2273,15 +2369,20 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
2273
2369
  const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
2274
2370
  const containerBound = binding.source === "container memory limit";
2275
2371
  const chunked = rowsLive !== null;
2276
- const recommendedChunk = chunked ? recommendedChunkFor(
2277
- maxAllowedMemory,
2372
+ const chunkHeldBudget = /* @__PURE__ */ __name((rows) => maxAllowedMemory - (includeExternal ? heldForChunk(rows) : 0), "chunkHeldBudget");
2373
+ const chunkFitting = /* @__PURE__ */ __name((rows) => recommendedChunkFor(
2374
+ chunkHeldBudget(rows),
2278
2375
  symbolTableSize,
2279
2376
  maxRows,
2280
2377
  totalRows,
2281
2378
  columnCount,
2282
2379
  liveRowsPerChunk,
2283
2380
  includeExternal
2284
- ) : 0;
2381
+ ), "chunkFitting");
2382
+ const callersChunk = chunked ? Math.max(1, Math.floor(rowsLive / Math.max(1, liveRowsPerChunk))) : 0;
2383
+ const firstChunk = chunked ? chunkFitting(callersChunk) : 0;
2384
+ const overChunk = chunked ? chunkFitting(firstChunk) : 0;
2385
+ const recommendedChunk = chunked ? Math.max(firstChunk, chunkFitting(overChunk)) : 0;
2285
2386
  const knob = chunked ? "chunkSize" : "limit";
2286
2387
  const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
2287
2388
  const nothingFits = recommendedValue === 0;
@@ -2293,31 +2394,75 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
2293
2394
  } else {
2294
2395
  advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
2295
2396
  }
2296
- throw new exports.QvdValidationError(
2297
- `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
2298
- {
2299
- file: filePath,
2300
- symbolTableSize,
2301
- symbolTableSizeMB: sizeMB,
2302
- estimatedMemoryMB: estimatedMB,
2303
- availableMemoryMB: availableMB,
2304
- heapLimitMB,
2305
- reportedHeapLimitMB,
2306
- availableRamMB,
2307
- limitingFactor,
2308
- limitingScope,
2309
- memoryBudget: budget.candidates,
2310
- memoryObserved: budget.observed,
2311
- columnCount,
2312
- totalRows,
2313
- maxRows,
2314
- recommendedMaxRows,
2315
- // Only present when a chunk size is what overflowed, so a caller cannot mistake one
2316
- // recommendation for the other.
2317
- ...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
2397
+ const suggestions = [];
2398
+ if (!nothingFits) {
2399
+ suggestions.push({ option: knob, value: recommendedValue });
2400
+ }
2401
+ if (!containerBound) {
2402
+ let needed = Math.ceil(estimatedMemory / safetyFactor / (1024 * 1024));
2403
+ while (Math.max(needed * 1024 * 1024, MINIMUM_BUDGET_BYTES) * safetyFactor < estimatedMemory) {
2404
+ needed += 1;
2318
2405
  }
2319
- );
2406
+ suggestions.push({ nodeOption: "--max-old-space-size", value: needed });
2407
+ }
2408
+ return {
2409
+ fits: false,
2410
+ reason: "memory",
2411
+ estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
2412
+ budget: budgetOf(budget, tightest, safetyFactor),
2413
+ // The symbol term is six times the bytes on disk, an overhead measured across files rather than
2414
+ // derived, so any read with symbols in it is an estimate and says so. Only a read that decodes
2415
+ // nothing can be exact.
2416
+ exact: symbolTableSize === 0,
2417
+ suggestions,
2418
+ refusal: {
2419
+ message: `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
2420
+ context: {
2421
+ symbolTableSize,
2422
+ symbolTableSizeMB: sizeMB,
2423
+ estimatedMemoryMB: estimatedMB,
2424
+ availableMemoryMB: availableMB,
2425
+ heapLimitMB,
2426
+ reportedHeapLimitMB,
2427
+ availableRamMB,
2428
+ limitingFactor,
2429
+ limitingScope,
2430
+ memoryBudget: budget.candidates,
2431
+ memoryObserved: budget.observed,
2432
+ columnCount,
2433
+ totalRows,
2434
+ maxRows,
2435
+ recommendedMaxRows,
2436
+ // Only present when a chunk size is what overflowed, so a caller cannot mistake one
2437
+ // recommendation for the other.
2438
+ ...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
2439
+ }
2440
+ }
2441
+ };
2320
2442
  }
2443
+ return {
2444
+ fits: true,
2445
+ estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
2446
+ budget: budgetOf(budget, tightest, safetyFactor),
2447
+ exact: symbolTableSize === 0,
2448
+ suggestions: []
2449
+ };
2450
+ }
2451
+ function answerOf(answer) {
2452
+ const { refusal, ...rest } = answer;
2453
+ return rest;
2454
+ }
2455
+ function budgetOf(budget, tightest, safetyFactor) {
2456
+ const processLimit = budget.candidates.find((candidate) => candidate.source !== "V8 heap limit");
2457
+ return {
2458
+ heapBytes: usableOldSpaceLimit(),
2459
+ processBytes: processLimit ? processLimit.bytes : null,
2460
+ bound: tightest.heapOnly ? "heap" : "process",
2461
+ safetyFactor,
2462
+ allowedBytes: tightest.allowed ?? tightest.bytes * safetyFactor,
2463
+ candidates: budget.candidates,
2464
+ observed: budget.observed
2465
+ };
2321
2466
  }
2322
2467
  function formatCount(value) {
2323
2468
  return value.toLocaleString("en-US");
@@ -2339,7 +2484,7 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount =
2339
2484
  `\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
2340
2485
  );
2341
2486
  }
2342
- var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
2487
+ var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES, noBytesHeld;
2343
2488
  var init_memoryUtils = __esm({
2344
2489
  "src/util/memoryUtils.js"() {
2345
2490
  init_QvdErrors();
@@ -2357,7 +2502,11 @@ var init_memoryUtils = __esm({
2357
2502
  __name(estimateMemoryUsage, "estimateMemoryUsage");
2358
2503
  __name(recommendedRowsFor, "recommendedRowsFor");
2359
2504
  __name(recommendedChunkFor, "recommendedChunkFor");
2505
+ noBytesHeld = Object.freeze({ held: 0, forRows: /* @__PURE__ */ __name(() => 0, "forRows"), forChunk: /* @__PURE__ */ __name(() => 0, "forChunk") });
2360
2506
  __name(validateMemoryAvailability, "validateMemoryAvailability");
2507
+ __name(checkMemory, "checkMemory");
2508
+ __name(answerOf, "answerOf");
2509
+ __name(budgetOf, "budgetOf");
2361
2510
  __name(formatCount, "formatCount");
2362
2511
  __name(warnLargeSymbolTable, "warnLargeSymbolTable");
2363
2512
  }
@@ -2476,6 +2625,7 @@ function validateSymbolTableSize(symbolTableLength, filePath, totalRows) {
2476
2625
  function validateFieldMetadata(field, symbolBufferLength, filePath) {
2477
2626
  const symbolsOffset = headerInteger(field["Offset"]);
2478
2627
  const symbolsLength = headerInteger(field["Length"]);
2628
+ const symbolCount = headerInteger(field["NoOfSymbols"]);
2479
2629
  if (isNaN(symbolsOffset) || !Number.isSafeInteger(symbolsOffset) || symbolsOffset < 0) {
2480
2630
  throw new exports.QvdCorruptedError("Invalid symbol offset", {
2481
2631
  field: field["FieldName"],
@@ -2492,6 +2642,14 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
2492
2642
  stage: "parseSymbolTable"
2493
2643
  });
2494
2644
  }
2645
+ if (isNaN(symbolCount) || !Number.isSafeInteger(symbolCount) || symbolCount < 0) {
2646
+ throw new exports.QvdCorruptedError("Invalid symbol count", {
2647
+ field: field["FieldName"],
2648
+ noOfSymbols: symbolCount,
2649
+ file: filePath,
2650
+ stage: "parseSymbolTable"
2651
+ });
2652
+ }
2495
2653
  if (symbolsOffset + symbolsLength > symbolBufferLength) {
2496
2654
  throw new exports.QvdCorruptedError("Symbol data extends beyond buffer", {
2497
2655
  field: field["FieldName"],
@@ -2747,12 +2905,25 @@ var init_validationUtils = __esm({
2747
2905
  __name(validateFieldBitMetadata, "validateFieldBitMetadata");
2748
2906
  }
2749
2907
  });
2750
-
2751
- // src/util/symbolParser.js
2752
- function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
2753
- const bufferLength = symbolBuffer.length;
2754
- const found = symbolBuffer.indexOf(0, from);
2755
- if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
2908
+ function nulFinder(area, { reach, rebaseAfter }) {
2909
+ assert4__default.default(reach - rebaseAfter > MAX_TEXT_BYTES, "A text search view must hold the longest text a symbol may have.");
2910
+ if (area.length <= reach) {
2911
+ return (from) => area.indexOf(0, from);
2912
+ }
2913
+ let base = 0;
2914
+ let view = area.subarray(0, reach);
2915
+ return (from) => {
2916
+ if (from - base > rebaseAfter) {
2917
+ base = from;
2918
+ view = area.subarray(base, Math.min(area.length, base + reach));
2919
+ }
2920
+ const found = view.indexOf(0, from - base);
2921
+ return found === -1 ? -1 : base + found;
2922
+ };
2923
+ }
2924
+ function textEnd(findNul, areaEnd, from, kind, fieldName, filePath, base) {
2925
+ const found = findNul(from);
2926
+ if ((found === -1 ? areaEnd : found) - from > MAX_TEXT_BYTES) {
2756
2927
  throw new exports.QvdCorruptedError(`${kind} exceeds maximum length`, {
2757
2928
  field: fieldName,
2758
2929
  maxLength: MAX_TEXT_BYTES,
@@ -2763,58 +2934,59 @@ function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
2763
2934
  if (found === -1) {
2764
2935
  throw new exports.QvdCorruptedError(`${kind} not null-terminated`, {
2765
2936
  field: fieldName,
2766
- pointer: bufferLength,
2767
- bufferSize: bufferLength,
2937
+ pointer: base + from,
2938
+ areaEnd: base + areaEnd,
2768
2939
  file: filePath,
2769
2940
  stage: "parseSymbolTable"
2770
2941
  });
2771
2942
  }
2772
2943
  return found;
2773
2944
  }
2774
- function overflow(message, pointer, bufferLength, fieldName, filePath) {
2945
+ function overflow(message, pointer, areaEnd, fieldName, filePath, base) {
2775
2946
  throw new exports.QvdCorruptedError(message, {
2776
2947
  field: fieldName,
2777
- pointer,
2778
- bufferSize: bufferLength,
2948
+ pointer: base + pointer,
2949
+ areaEnd: base + areaEnd,
2779
2950
  file: filePath,
2780
2951
  stage: "parseSymbolTable"
2781
2952
  });
2782
2953
  }
2783
- function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
2784
- const bufferLength = symbolBuffer.length;
2954
+ function parseFieldSymbols(symbolBuffer, start, end, symbolCount, keep, fieldName, filePath, search = TEXT_SEARCH, base = 0) {
2955
+ const area = symbolBuffer.subarray(0, end);
2956
+ const findNul = nulFinder(area, search);
2785
2957
  const numbers = [];
2786
2958
  const texts = [];
2787
2959
  let pointer = start;
2788
2960
  while (pointer < end) {
2789
- const typeByte = symbolBuffer[pointer++];
2961
+ const typeByte = area[pointer++];
2790
2962
  const decode = keep === null || keep.has(numbers.length);
2791
2963
  let number = null;
2792
2964
  let text = null;
2793
2965
  switch (typeByte) {
2794
2966
  case 1: {
2967
+ if (pointer + 4 > end) {
2968
+ overflow("Buffer overflow reading integer symbol", pointer, end, fieldName, filePath, base);
2969
+ }
2795
2970
  if (decode) {
2796
- if (pointer + 4 > bufferLength) {
2797
- overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
2798
- }
2799
- number = symbolBuffer.readInt32LE(pointer);
2971
+ number = area.readInt32LE(pointer);
2800
2972
  }
2801
2973
  pointer += 4;
2802
2974
  break;
2803
2975
  }
2804
2976
  case 2: {
2977
+ if (pointer + 8 > end) {
2978
+ overflow("Buffer overflow reading double symbol", pointer, end, fieldName, filePath, base);
2979
+ }
2805
2980
  if (decode) {
2806
- if (pointer + 8 > bufferLength) {
2807
- overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
2808
- }
2809
- number = symbolBuffer.readDoubleLE(pointer);
2981
+ number = area.readDoubleLE(pointer);
2810
2982
  }
2811
2983
  pointer += 8;
2812
2984
  break;
2813
2985
  }
2814
2986
  case 4: {
2815
- const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
2987
+ const terminator = textEnd(findNul, end, pointer, "String symbol", fieldName, filePath, base);
2816
2988
  if (decode) {
2817
- text = symbolBuffer.toString("utf8", pointer, terminator);
2989
+ text = area.toString("utf8", pointer, terminator);
2818
2990
  }
2819
2991
  pointer = terminator + 1;
2820
2992
  break;
@@ -2822,14 +2994,22 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
2822
2994
  case 5:
2823
2995
  case 6: {
2824
2996
  const numberBytes = typeByte === 5 ? 4 : 8;
2825
- if (pointer + numberBytes > bufferLength) {
2826
- const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
2827
- overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
2997
+ if (pointer + numberBytes > end) {
2998
+ const read = typeByte === 5 ? "dual integer symbol" : "dual double symbol";
2999
+ overflow(`Buffer overflow reading ${read}`, pointer, end, fieldName, filePath, base);
2828
3000
  }
2829
- const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
3001
+ const terminator = textEnd(
3002
+ findNul,
3003
+ end,
3004
+ pointer + numberBytes,
3005
+ "Dual string symbol",
3006
+ fieldName,
3007
+ filePath,
3008
+ base
3009
+ );
2830
3010
  if (decode) {
2831
- number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
2832
- text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
3011
+ number = typeByte === 5 ? area.readInt32LE(pointer) : area.readDoubleLE(pointer);
3012
+ text = area.toString("utf8", pointer + numberBytes, terminator);
2833
3013
  }
2834
3014
  pointer = terminator + 1;
2835
3015
  break;
@@ -2837,7 +3017,7 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
2837
3017
  default: {
2838
3018
  throw new exports.QvdParseError("Unknown symbol type byte", {
2839
3019
  typeByte: typeByte.toString(16),
2840
- offset: pointer - 1,
3020
+ offset: base + pointer - 1,
2841
3021
  file: filePath,
2842
3022
  stage: "parseSymbolTable"
2843
3023
  });
@@ -2846,16 +3026,28 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
2846
3026
  numbers.push(number);
2847
3027
  texts.push(text);
2848
3028
  }
3029
+ assert4__default.default(pointer === end, `The symbols of ${fieldName} were walked to byte ${pointer} of an area ending at ${end}.`);
3030
+ if (numbers.length !== symbolCount) {
3031
+ throw new exports.QvdCorruptedError("Symbol count mismatch", {
3032
+ field: fieldName,
3033
+ symbolCount: numbers.length,
3034
+ noOfSymbols: symbolCount,
3035
+ file: filePath,
3036
+ stage: "parseSymbolTable"
3037
+ });
3038
+ }
2849
3039
  return { numbers, texts };
2850
3040
  }
2851
- function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
2852
- return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
3041
+ function countFieldSymbols(symbolBuffer, start, end, symbolCount, fieldName, filePath, base = 0) {
3042
+ return parseFieldSymbols(symbolBuffer, start, end, symbolCount, DECODE_NOTHING, fieldName, filePath, void 0, base).numbers.length;
2853
3043
  }
2854
- var MAX_TEXT_BYTES, DECODE_NOTHING;
3044
+ var MAX_TEXT_BYTES, TEXT_SEARCH, DECODE_NOTHING;
2855
3045
  var init_symbolParser = __esm({
2856
3046
  "src/util/symbolParser.js"() {
2857
3047
  init_QvdErrors();
2858
3048
  MAX_TEXT_BYTES = 1048576;
3049
+ TEXT_SEARCH = Object.freeze({ reach: 2 ** 31 - 1, rebaseAfter: 2 ** 30 });
3050
+ __name(nulFinder, "nulFinder");
2859
3051
  __name(textEnd, "textEnd");
2860
3052
  __name(overflow, "overflow");
2861
3053
  __name(parseFieldSymbols, "parseFieldSymbols");
@@ -3399,7 +3591,17 @@ async function parseHeaderXml(text, file, stage) {
3399
3591
  }
3400
3592
  return parsed;
3401
3593
  }
3402
- var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
3594
+ function symbolBytesOf(selected, symbolTableLength) {
3595
+ const areaBytes = selected.map((field) => headerInteger(field["Length"]));
3596
+ return areaBytes.every((bytes) => Number.isSafeInteger(bytes) && bytes >= 0) ? Math.min(
3597
+ symbolTableLength,
3598
+ areaBytes.reduce((sum, bytes) => sum + bytes, 0)
3599
+ ) : symbolTableLength;
3600
+ }
3601
+ function readPasses(analysisAhead) {
3602
+ return analysisAhead ? 2 : 1;
3603
+ }
3604
+ var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, SLICE_BYTES, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
3403
3605
  var init_QvdFileReader = __esm({
3404
3606
  "src/QvdFileReader.js"() {
3405
3607
  init_QvdDataFrame();
@@ -3417,9 +3619,12 @@ var init_QvdFileReader = __esm({
3417
3619
  MAX_HEADER_SIZE = 16 * 1024 * 1024;
3418
3620
  READ_CHUNK_SIZE = 512 * 1024 * 1024;
3419
3621
  ANALYSIS_SLICE_ROWS = 65536;
3622
+ SLICE_BYTES = 16 * 1024 * 1024;
3420
3623
  COUNT_SYMBOLS_PAST = 65536;
3421
3624
  __name(chunksFrom, "chunksFrom");
3422
3625
  __name(parseHeaderXml, "parseHeaderXml");
3626
+ __name(symbolBytesOf, "symbolBytesOf");
3627
+ __name(readPasses, "readPasses");
3423
3628
  exports.QvdFileReader = class {
3424
3629
  static {
3425
3630
  __name(this, "QvdFileReader");
@@ -3448,6 +3653,10 @@ var init_QvdFileReader = __esm({
3448
3653
  * above which a lazy load switches to the two-pass filtering path. The default of 50MB is
3449
3654
  * the point where the extra analysis pass pays for itself; lower it to use filtering on
3450
3655
  * smaller files, raise it to keep the simpler single-pass read for longer.
3656
+ * @param {number} [options.sliceBytes=16777216] The most bytes of records a read holds at a time. An
3657
+ * option rather than a constant for the reason `symbolFilteringThreshold` is one: so that a test can
3658
+ * cross the boundaries between slices in a small file. There is no other reason to change it, and it
3659
+ * is not one of the options a read through `QvdDataFrame` or `QvdColumnTable` passes on.
3451
3660
  * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
3452
3661
  * appear. Null reads every field, in file order. An unknown or repeated name is refused.
3453
3662
  * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
@@ -3472,6 +3681,7 @@ var init_QvdFileReader = __esm({
3472
3681
  allowedDir,
3473
3682
  memorySafetyFactor = 0.8,
3474
3683
  symbolFilteringThreshold = 50 * 1024 * 1024,
3684
+ sliceBytes = SLICE_BYTES,
3475
3685
  materialisesRows = true,
3476
3686
  fields = null,
3477
3687
  duals,
@@ -3487,6 +3697,14 @@ var init_QvdFileReader = __esm({
3487
3697
  this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
3488
3698
  this._memorySafetyFactor = memorySafetyFactor;
3489
3699
  this._symbolFilteringThreshold = symbolFilteringThreshold;
3700
+ if (!Number.isSafeInteger(sliceBytes) || sliceBytes <= 0) {
3701
+ throw new exports.QvdValidationError("sliceBytes must be a positive integer", {
3702
+ provided: sliceBytes,
3703
+ type: typeof sliceBytes,
3704
+ file: this._path
3705
+ });
3706
+ }
3707
+ this._sliceBytes = sliceBytes;
3490
3708
  if (onProgress !== void 0 && typeof onProgress !== "function") {
3491
3709
  throw new exports.QvdValidationError("onProgress must be a function", {
3492
3710
  provided: onProgress,
@@ -3504,7 +3722,11 @@ var init_QvdFileReader = __esm({
3504
3722
  this._requestedFields = fields === void 0 ? null : fields;
3505
3723
  this._onProgress = onProgress;
3506
3724
  this._signal = signal;
3507
- this._buffer = null;
3725
+ this._headerBuffer = null;
3726
+ this._handle = null;
3727
+ this._failed = null;
3728
+ this._reading = false;
3729
+ this._symbolAreas = null;
3508
3730
  this._headerOffset = null;
3509
3731
  this._symbolTableOffset = null;
3510
3732
  this._indexTableOffset = null;
@@ -3515,7 +3737,6 @@ var init_QvdFileReader = __esm({
3515
3737
  this._symbolTable = null;
3516
3738
  this._indexColumns = null;
3517
3739
  this._rowsDecoded = 0;
3518
- this._bufferFirstRow = 0;
3519
3740
  this._fileSize = null;
3520
3741
  this._headerMatchesFile = false;
3521
3742
  }
@@ -3554,54 +3775,137 @@ var init_QvdFileReader = __esm({
3554
3775
  }
3555
3776
  }
3556
3777
  /**
3557
- * Reads the binary data of the QVD file.
3558
- *
3559
- * A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
3560
- * needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
3561
- * over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
3562
- *
3563
- * The saving is in the index table and the rows, not in the symbol table, which is read in
3564
- * full whatever the window because a stored index in any row can address any symbol. So the
3565
- * gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
3566
- * values there is very little to save, which is what `symbolFilteringThreshold` and the
3567
- * two-pass path exist for.
3568
- *
3569
- * Algorithm for a windowed read:
3570
- * 1. Read the file a chunk at a time until the XML header delimiter is found
3571
- * 2. Parse header to determine symbol table and index table locations
3572
- * 3. Calculate bytes needed: header + full symbol table + partial index table
3573
- * 4. Read only those calculated bytes, by position
3574
- * 5. Rest of parsing proceeds normally with limited data
3575
- *
3576
- * WHY THIS APPROACH:
3577
- * - Symbol table must be fully loaded (contains all unique values)
3578
- * - Index table can be partially loaded (only rows we need)
3579
- * - Reading chunks to find the header is efficient for unknown header sizes
3580
- * - Direct byte-range reading for remaining data is fastest
3778
+ * Opens the file and reads its header, and for a read of rows checks that the read can be made.
3779
+ *
3780
+ * Nothing past the header is read here. The selected fields' symbols and the records are read by position
3781
+ * as they are parsed - see `_symbolAreaOf` and `_forEachSlice` - so no read holds the file in one buffer
3782
+ * (#122). A read of rows therefore leaves the file open, and whatever started the read closes it with
3783
+ * `_closeFile` once the last record it needs is decoded; a header-only read closes it here.
3784
+ *
3785
+ * A windowed read - anything with `offset`, `limit` or `maxRows` - reads the header, the symbols of the
3786
+ * fields it selects, and the window's records, and no byte between. Measured on
3787
+ * `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows over 20 fields: the last thousand rows take 19 ms
3788
+ * against 636 ms for the whole file. A selected field's area is read in full whatever the window,
3789
+ * because a stored index in any row can address any of that field's symbols; the areas of the fields
3790
+ * `fields` leaves out are not read at all. So a window's gain scales with how much of the file is
3791
+ * rows: on a file whose bytes are mostly distinct values of the fields it reads there is very little
3792
+ * to save, which is what `symbolFilteringThreshold` and the two-pass path exist for.
3581
3793
  *
3582
3794
  * All of it goes through one handle, opened once, on the file the containment check approved. See
3583
3795
  * `chunksFrom` and `openChecked` for why a read no longer opens the path more than once.
3584
3796
  *
3585
- * A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
3586
- * table from the front of the file, and the window's records from wherever they sit. The bytes
3587
- * between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
3588
- * fixture a 0.4MB read rather than a 38MB one.
3589
- *
3590
3797
  * @param {QvdRowWindow} window The rows to read.
3591
- * @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
3592
- * symbol and index tables on disk. This is the metadata-only path: the header is a few
3593
- * kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
3594
- * for a 40GB one.
3798
+ * @param {boolean} [headerOnly=false] Stop once the XML header has been read, and close the file. This
3799
+ * is the metadata-only path: the header is a few kilobytes whatever the file's size, so reading a
3800
+ * schema costs the same for a 40MB file as for a 40GB one.
3595
3801
  * @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
3596
3802
  * that is fewer than the window covers - see `_prepare`.
3597
3803
  * @private
3598
3804
  */
3599
3805
  async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
3806
+ assert4__default.default(this._reading, "A read opens the QVD file only once it has started, through _startRead.");
3807
+ this._symbolTable = null;
3808
+ this._indexColumns = null;
3809
+ this._rowsDecoded = 0;
3600
3810
  this._throwIfAborted();
3601
3811
  this._emitProgress("read", 0, 1);
3602
3812
  const failed = rethrowAsIoError(this._path, "read");
3603
3813
  const handle = await openChecked(checkPath(this._path, this._allowedDir), "read", failed);
3604
- await closeAfter(handle, failed, () => this._readFrom(handle, window, headerOnly, liveRows, failed));
3814
+ if (headerOnly) {
3815
+ await closeAfter(handle, failed, () => this._readFrom(handle, window, true, liveRows, failed));
3816
+ return;
3817
+ }
3818
+ this._handle = handle;
3819
+ this._failed = failed;
3820
+ try {
3821
+ await this._readFrom(handle, window, false, liveRows, failed);
3822
+ } catch (error) {
3823
+ await this._closeFile(true);
3824
+ throw error;
3825
+ }
3826
+ }
3827
+ /**
3828
+ * Starts a read on this reader, refusing it while another is under way.
3829
+ *
3830
+ * A read of rows holds the file, and what it has read of it, on the reader until it ends. A second read
3831
+ * started meanwhile - `load()` while an iteration is suspended, say - would take over that state and
3832
+ * leave the first read's file open. Reads one after another are fine. Called before a read takes charge
3833
+ * of closing the file, so that refusing the second read cannot close the first one's.
3834
+ *
3835
+ * The first thing every read does, and synchronous: the flag is set before the read's first `await`, so
3836
+ * two reads started together - `Promise.all([reader.load(), reader.load()])` - cannot both pass. The check
3837
+ * used to be of the handle, which is set only once the file has opened, and both did: the second read's
3838
+ * handle replaced the first's, which was never closed, and the first read to finish closed the file the
3839
+ * other was still reading.
3840
+ *
3841
+ * @throws {QvdValidationError} If a read is under way.
3842
+ * @private
3843
+ */
3844
+ _startRead() {
3845
+ if (this._reading) {
3846
+ throw new exports.QvdValidationError("The reader is already reading this file: finish that read first", {
3847
+ file: this._path
3848
+ });
3849
+ }
3850
+ this._reading = true;
3851
+ }
3852
+ /**
3853
+ * Ends the read `_startRead` began: closes its file, if it still holds one, and lets the next read start.
3854
+ *
3855
+ * @param {boolean} failing Whether the read is already throwing - see `_closeFile`.
3856
+ * @private
3857
+ */
3858
+ async _endRead(failing) {
3859
+ try {
3860
+ await this._closeFile(failing);
3861
+ } finally {
3862
+ this._reading = false;
3863
+ }
3864
+ }
3865
+ /**
3866
+ * Closes the file a read of rows opened, and drops what it had read of it.
3867
+ *
3868
+ * The rule `closeAfter` follows: a close that fails is reported only when the read succeeded, so it can
3869
+ * never replace the error that says what went wrong. After a successful read it is the only news.
3870
+ *
3871
+ * @param {boolean} failing Whether the read is already throwing.
3872
+ * @private
3873
+ */
3874
+ async _closeFile(failing) {
3875
+ const handle = this._handle;
3876
+ const failed = this._failed;
3877
+ this._handle = null;
3878
+ this._failed = null;
3879
+ this._symbolAreas = null;
3880
+ if (handle === null || failed === null) {
3881
+ return;
3882
+ }
3883
+ if (failing) {
3884
+ await handle.close().catch(() => {
3885
+ });
3886
+ return;
3887
+ }
3888
+ await handle.close().catch(failed);
3889
+ }
3890
+ /**
3891
+ * Runs a read, the only one under way on this reader, and closes its file when it ends, however it ends.
3892
+ *
3893
+ * @template T
3894
+ * @param {() => Promise<T>} read The read, from opening the file to its last record.
3895
+ * @return {Promise<T>} What it returned.
3896
+ * @private
3897
+ */
3898
+ async _closingAfter(read) {
3899
+ this._startRead();
3900
+ let result;
3901
+ try {
3902
+ result = await read();
3903
+ } catch (error) {
3904
+ await this._endRead(true);
3905
+ throw error;
3906
+ }
3907
+ await this._endRead(false);
3908
+ return result;
3605
3909
  }
3606
3910
  /**
3607
3911
  * Reads what `_readData` was asked for, through the handle it opened.
@@ -3662,51 +3966,61 @@ var init_QvdFileReader = __esm({
3662
3966
  const indexTableOffset = symbolTableOffset + symbolTableLength;
3663
3967
  const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
3664
3968
  const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
3665
- if (headerOnly) {
3666
- this._buffer = headerBuffer.subarray(0, headerEndIndex);
3667
- this._emitProgress("read", 1, 1);
3668
- return;
3669
- }
3670
- const columnCount = selectFields(headerFields, this._requestedFields, this._path).length;
3969
+ const { size: fileSize } = await handle.stat().catch(failed);
3970
+ this._fileSize = fileSize;
3971
+ this._headerMatchesFile = false;
3671
3972
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
3672
3973
  (value) => Number.isSafeInteger(value) && value >= 0
3673
3974
  );
3674
3975
  if (headerNumbersUsable) {
3675
- const { size: fileSize2 } = await handle.stat().catch(failed);
3676
- this._fileSize = fileSize2;
3677
- this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize2;
3976
+ this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
3977
+ }
3978
+ if (headerOnly) {
3979
+ this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
3980
+ this._emitProgress("read", 1, 1);
3981
+ return;
3678
3982
  }
3983
+ this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
3984
+ const selected = selectFields(headerFields, this._requestedFields, this._path);
3985
+ const columnCount = selected.length;
3986
+ const symbolBytes = symbolBytesOf(selected, symbolTableLength);
3679
3987
  const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
3680
3988
  const windowRows = resolved.limit;
3681
3989
  if (headerNumbersUsable && this._headerMatchesFile) {
3682
3990
  validateMemoryAvailability(
3683
- symbolTableLength,
3991
+ symbolBytes,
3684
3992
  windowRows,
3685
3993
  totalRows,
3686
3994
  this._path,
3687
3995
  this._memorySafetyFactor,
3688
3996
  columnCount,
3689
3997
  this._materialisesRows,
3690
- liveRows
3998
+ liveRows,
3999
+ this._bytesHeld(
4000
+ symbolBytes,
4001
+ windowRows,
4002
+ recordSize,
4003
+ liveRows,
4004
+ this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)
4005
+ ),
4006
+ // What it reads, which is not what it holds - the records go through one buffer and are not
4007
+ // kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
4008
+ //
4009
+ // Twice over where the symbol-usage pass is still ahead of it: that pass reads the window's
4010
+ // records to find which symbols the rows use, and the decode then reads them again. Counted
4011
+ // once, the figure understated the I/O of exactly the reads that do the most of it.
4012
+ symbolBytes + readPasses(this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) * windowRows * recordSize
3691
4013
  );
3692
4014
  }
3693
4015
  if (window.offset === 0 && window.limit === null) {
3694
- this._buffer = await handle.readFile().catch(failed);
3695
- this._fileSize = this._buffer.length;
3696
- this._bufferFirstRow = 0;
3697
4016
  this._emitProgress("read", 1, 1);
3698
4017
  return;
3699
4018
  }
3700
4019
  const rowsToLoad = windowRows;
3701
- validateSymbolTableSizeEarly(symbolTableLength, this._path);
4020
+ validateSymbolTableSizeEarly(symbolBytes, this._path);
3702
4021
  validateRecordSize(recordSize, this._path, "readData");
3703
4022
  validateRecordCount(totalRows, this._path, "readData");
3704
- const skippedIndexBytes = resolved.offset * recordSize;
3705
- const indexTableBytesToRead = rowsToLoad * recordSize;
3706
- const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
3707
- const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
3708
- const { size: fileSize } = await handle.stat().catch(failed);
3709
- this._fileSize = fileSize;
4023
+ const fileBytesRequired = indexTableOffset + (resolved.offset + rowsToLoad) * recordSize;
3710
4024
  if (fileBytesRequired > fileSize) {
3711
4025
  throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
3712
4026
  file: this._path,
@@ -3715,47 +4029,35 @@ var init_QvdFileReader = __esm({
3715
4029
  stage: "readData"
3716
4030
  });
3717
4031
  }
3718
- this._buffer = Buffer.alloc(totalBytesToRead);
3719
- await this._readRange(handle, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
3720
- if (indexTableBytesToRead > 0) {
3721
- await this._readRange(
3722
- handle,
3723
- indexTableOffset,
3724
- indexTableBytesToRead,
3725
- indexTableOffset + skippedIndexBytes,
3726
- fileSize,
3727
- fileBytesRequired
3728
- );
3729
- }
3730
- this._bufferFirstRow = resolved.offset;
3731
4032
  this._emitProgress("read", 1, 1);
3732
4033
  }
3733
4034
  /**
3734
- * Reads one byte range of the file into the buffer.
4035
+ * Reads one byte range of the open file into a buffer.
3735
4036
  *
3736
4037
  * Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
3737
4038
  * 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
3738
4039
  * try/catch can intercept.
3739
4040
  *
3740
- * @param {import('fs/promises').FileHandle} fd The open file.
3741
- * @param {number} bufferOffset Where in the buffer to write.
4041
+ * @param {Buffer} target The buffer to read into.
4042
+ * @param {number} targetOffset Where in it to write.
3742
4043
  * @param {number} byteCount How many bytes to read.
3743
4044
  * @param {number} filePosition Where in the file to read from.
3744
- * @param {number} fileSize The file's size, for the error.
3745
- * @param {number} requiredBytes Bytes the whole read needs, for the error.
4045
+ * @param {number} requiredBytes How far into the file the read has to reach, for the error.
4046
+ * @throws {QvdCorruptedError} If the file ends before the range does.
3746
4047
  * @private
3747
4048
  */
3748
- async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
3749
- assert3__default.default(this._buffer, "The read buffer has not been allocated.");
3750
- const failed = rethrowAsIoError(this._path, "read");
4049
+ async _readAt(target, targetOffset, byteCount, filePosition, requiredBytes) {
4050
+ assert4__default.default(this._handle && this._failed, "The QVD file is not open.");
4051
+ const handle = this._handle;
4052
+ const failed = this._failed;
3751
4053
  let done = 0;
3752
4054
  while (done < byteCount) {
3753
4055
  const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
3754
- const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done).catch(failed);
4056
+ const { bytesRead } = await handle.read(target, targetOffset + done, length, filePosition + done).catch(failed);
3755
4057
  if (bytesRead === 0) {
3756
4058
  throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
3757
4059
  file: this._path,
3758
- fileSize,
4060
+ fileSize: this._fileSize,
3759
4061
  // Two numbers, because they stopped being the same one when a window began reading two
3760
4062
  // ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
3761
4063
  // file it gave up. Reporting the position under the name of the count made a windowed
@@ -3770,12 +4072,279 @@ var init_QvdFileReader = __esm({
3770
4072
  done += bytesRead;
3771
4073
  }
3772
4074
  }
4075
+ /**
4076
+ * Bytes of the file a read holds outside the heap while it works, beside the codes the guard counts for
4077
+ * itself: the symbol areas it reads, and the one buffer its records come through.
4078
+ *
4079
+ * The areas are counted whole, although the read lets each range go once its fields are parsed, because
4080
+ * two ranges are both live when a field of one is parsed between two fields of the other - the order the
4081
+ * caller asked for the fields decides it, so the sum is what holds in every order. The slice is the
4082
+ * buffer `_forEachSlice` will allocate, sized by `_sliceRowsFor` so that the charge and the allocation
4083
+ * are one expression rather than two that agree today.
4084
+ *
4085
+ * @param {number} symbolBytes Bytes of symbols the read will read.
4086
+ * @param {number} rows Records it will read.
4087
+ * @param {number} recordSize Bytes per record.
4088
+ * @return {number} Bytes.
4089
+ * @private
4090
+ */
4091
+ _bytesHeldBy(symbolBytes, rows, recordSize) {
4092
+ const usable = Number.isSafeInteger(rows) && Number.isSafeInteger(recordSize) && rows > 0 && recordSize > 0;
4093
+ return symbolBytes + (usable ? this._sliceRowsFor(rows, recordSize) * recordSize : 0);
4094
+ }
4095
+ /**
4096
+ * Records the one buffer holds while a read of `rowCount` records goes through it.
4097
+ *
4098
+ * A slice is `sliceBytes` of records, rounded down to a whole record, or every record the read has left
4099
+ * when that is fewer - and at least one, since a read of a record wider than `sliceBytes` still has to
4100
+ * hold that record. The single definition: `_forEachSlice` allocates from it and the memory guard is
4101
+ * charged from it, so a change to how a read slices cannot leave the guard pricing the old rule.
4102
+ *
4103
+ * @param {number} rowCount Records the read will read.
4104
+ * @param {number} recordSize Bytes per record.
4105
+ * @return {number} Records in one slice.
4106
+ * @private
4107
+ */
4108
+ _sliceRowsFor(rowCount, recordSize) {
4109
+ return Math.max(1, Math.min(rowCount, Math.floor(this._sliceBytes / Math.max(1, recordSize))));
4110
+ }
4111
+ /**
4112
+ * What a read holds in bytes of the file, for the memory guard: what it holds now, and what a read
4113
+ * following either piece of advice a refusal can carry would hold instead.
4114
+ *
4115
+ * The two knobs are not the same knob, which is why there are two functions rather than one. A smaller
4116
+ * `limit` is a smaller window, so every record the read touches is one of fewer - the symbol-usage pass
4117
+ * included, since it reads the window. A smaller `chunkSize` leaves the window exactly where it is and
4118
+ * only changes how much of it is decoded at a time, so a read with that pass still ahead of it holds the
4119
+ * window's slice however small the chunk. Priced with `forRows`, such a chunk was charged for the records
4120
+ * of one chunk and then held sixteen megabytes more than that.
4121
+ *
4122
+ * @param {number} symbolBytes Bytes of symbols the read will read.
4123
+ * @param {number} windowRows Rows the read covers.
4124
+ * @param {number} recordSize Bytes per record.
4125
+ * @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
4126
+ * @param {boolean} analysisAhead Whether the symbol-usage pass has still to run.
4127
+ * @return {{held: number, forRows: (rows: number) => number, forChunk: (rows: number) => number}} What
4128
+ * this read holds, what a read of so many rows would hold, and what one reading so many rows a chunk
4129
+ * would hold.
4130
+ * @private
4131
+ */
4132
+ _bytesHeld(symbolBytes, windowRows, recordSize, liveRows, analysisAhead) {
4133
+ return {
4134
+ held: this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, liveRows, analysisAhead), recordSize),
4135
+ // A window of so many rows reads so many records at a time, and the pass that reads it ahead of the
4136
+ // decode reads the same rows, so the buffer is sized from the rows either way.
4137
+ forRows: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, rows, recordSize), "forRows"),
4138
+ // A chunk of so many rows, over this read's window - which is what `chunkSize` changes and what it
4139
+ // leaves alone. `_recordsAtOnce` is what answers that, given a chunk size as the rows held at once.
4140
+ forChunk: /* @__PURE__ */ __name((rows) => this._bytesHeldBy(symbolBytes, this._recordsAtOnce(windowRows, { rows, perChunk: 1 }, analysisAhead), recordSize), "forChunk")
4141
+ };
4142
+ }
4143
+ /**
4144
+ * Whether a read takes the symbol-usage pass, which reads the window's records before the symbol table
4145
+ * is parsed and so before the first chunk is built.
4146
+ *
4147
+ * The one statement of the condition. `_prepare` asks it to decide, and `_readData` asks it before the
4148
+ * header has been parsed, to know what to charge the memory guard: a read with the pass ahead of it
4149
+ * holds a whole slice of records, and a read without it holds only what it reads at a time. Said in two
4150
+ * places, the two would drift and a read would be charged for one path and take the other - which fails
4151
+ * open for a chunked read, and that is the direction the guard exists to prevent.
4152
+ *
4153
+ * Any window that does not cover the whole file is a candidate, which includes one bounded by its offset
4154
+ * rather than by its limit. Covering every row rules it out, however the window was spelled -
4155
+ * `{offset: 0, limit: n}` over all n rows, or an `iterate` of them. Such a read needs every symbol any
4156
+ * row uses, which is what `estimateMemoryUsage` assumes for a full read as well, so the pass has nothing
4157
+ * to filter. It is not free: since the records are read as they are decoded rather than held in one
4158
+ * buffer, the pass reads the window's records and the decode then reads them again. A window that really
4159
+ * is a window pays that for the symbols it saves parsing; a window that is a full read in disguise paid
4160
+ * it for nothing.
4161
+ *
4162
+ * The threshold is an option rather than a constant so this path can be exercised with a small fixture:
4163
+ * it is the most intricate code in the reader, and the only files large enough to reach the 50MB default
4164
+ * are ones no repository should be carrying around. It measures the whole table rather than the areas a
4165
+ * read selects, because what the pass saves is parsing work across the table.
4166
+ *
4167
+ * @param {QvdRowWindow} window The window as the caller spelled it.
4168
+ * @param {{offset: number, limit: number}} resolved Where it lands in this file.
4169
+ * @param {number} totalRows Rows the file declares.
4170
+ * @param {number} symbolTableLength The symbol table's declared length.
4171
+ * @return {boolean} Whether the pass will run.
4172
+ * @private
4173
+ */
4174
+ _analysisWouldRun(window, resolved, totalRows, symbolTableLength) {
4175
+ return resolved.limit < totalRows && (window.limit !== null || window.offset > 0) && symbolTableLength > this._symbolFilteringThreshold;
4176
+ }
4177
+ /**
4178
+ * Records a read holds at one time, which is what its record buffer is sized from.
4179
+ *
4180
+ * A slice holds `sliceBytes` of records, or every record the read has left to read when that is fewer -
4181
+ * so what it costs depends on how many a read asks for at a time, not on how many it covers. An
4182
+ * iteration asks for a chunk: `iterate({limit: 20_000_000, chunkSize: 1000})` reads a thousand records at
4183
+ * a time however many its window covers, and charging it a full slice would refuse it for 16 MiB it never
4184
+ * allocates. The symbol-usage pass is the exception, because it reads the whole window in slices of its
4185
+ * own before the first chunk is built, so a read that still has that pass ahead of it is charged for it.
4186
+ *
4187
+ * @param {number} windowRows Rows the read covers.
4188
+ * @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
4189
+ * @param {boolean} analysisAhead Whether the symbol-usage pass has still to run.
4190
+ * @return {number} Records read at one time.
4191
+ * @private
4192
+ */
4193
+ _recordsAtOnce(windowRows, liveRows, analysisAhead) {
4194
+ const chunkRows = liveRows === null ? windowRows : Math.max(1, Math.floor(liveRows.rows / Math.max(1, liveRows.perChunk)));
4195
+ return analysisAhead ? Math.max(windowRows, chunkRows) : chunkRows;
4196
+ }
4197
+ /**
4198
+ * The symbol table's length, as much of it as the file holds: what the header declares, cut short where
4199
+ * the file ends. Known before a byte of the table is read, so everything that can refuse the table is
4200
+ * checked on this, before the table is allocated.
4201
+ *
4202
+ * A file that ends inside its symbol table is measured to where it ends, and the fields whose areas it cut
4203
+ * short are refused as `Symbol data extends beyond buffer` when their metadata is checked - what a
4204
+ * whole-file read has always said of such a file. A window has refused it already, before reading anything.
4205
+ *
4206
+ * @return {number} Bytes.
4207
+ * @private
4208
+ */
4209
+ _symbolTableLength() {
4210
+ assert4__default.default(
4211
+ this._symbolTableOffset !== null && this._indexTableOffset !== null && this._fileSize !== null,
4212
+ "The QVD file header has not been parsed before its symbol table was measured."
4213
+ );
4214
+ const declared = this._indexTableOffset - this._symbolTableOffset;
4215
+ return Math.max(0, Math.min(declared, this._fileSize - this._symbolTableOffset));
4216
+ }
4217
+ /**
4218
+ * Where each selected field's symbols are, as ranges of the symbol table this read will read.
4219
+ *
4220
+ * A field's `Offset` and `Length` say exactly where its symbols are, so a read of some of a file's fields
4221
+ * has no reason to read the areas of the rest (#122). Qlik writes the areas one after another in field
4222
+ * order, so ranges that touch are merged: a read of every field is one range, and so is a read of fields
4223
+ * that happen to be neighbours. A read of one field of twenty reads that field's area alone.
4224
+ *
4225
+ * Built once per read, from the fields the read selected, and each field's metadata is checked as it is
4226
+ * added - a range is arithmetic on `Offset` and `Length`, and those have to be inside the table first.
4227
+ * `_parseSymbolTable` checks every field of the file, selected or not, before it parses any.
4228
+ *
4229
+ * @return {{ranges: Array<{start: number, end: number, fields: number, buffer: Buffer|null}>,
4230
+ * byField: Map<any, {range: {start: number, end: number, fields: number, buffer: Buffer|null},
4231
+ * start: number, end: number}>}} The ranges, and where in its range each field's area sits.
4232
+ * @private
4233
+ */
4234
+ _symbolAreaPlan() {
4235
+ if (this._symbolAreas !== null) {
4236
+ return this._symbolAreas;
4237
+ }
4238
+ assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved before their symbols were read.");
4239
+ const tableLength = this._symbolTableLength();
4240
+ const areas = this._selectedFields.map((field) => {
4241
+ validateFieldMetadata(field, tableLength, this._path);
4242
+ const start = headerInteger(field["Offset"]);
4243
+ return { field, start, end: start + headerInteger(field["Length"]) };
4244
+ });
4245
+ const ranges = [];
4246
+ const byField = /* @__PURE__ */ new Map();
4247
+ for (const area of [...areas].sort((a, b) => a.start - b.start)) {
4248
+ const last = ranges.at(-1);
4249
+ const range = last !== void 0 && area.start <= last.end ? last : { start: area.start, end: area.end, fields: 0, buffer: null };
4250
+ if (range !== last) {
4251
+ ranges.push(range);
4252
+ }
4253
+ range.end = Math.max(range.end, area.end);
4254
+ range.fields += 1;
4255
+ byField.set(area.field, { range, start: area.start, end: area.end });
4256
+ }
4257
+ this._symbolAreas = { ranges, byField };
4258
+ return this._symbolAreas;
4259
+ }
4260
+ /**
4261
+ * One field's symbols, as bytes: the range that holds them, read from the open file the first time a field
4262
+ * of that range needs it.
4263
+ *
4264
+ * @param {any} field The field, one this read selected.
4265
+ * @return {Promise<{buffer: Buffer, start: number, end: number, base: number}>} Its area, as a range of
4266
+ * `buffer`, with where that buffer starts in the symbol table - what an error adds back to say where a
4267
+ * damaged symbol is in the file, rather than where it is in the bytes this read happened to read.
4268
+ * @throws {QvdValidationError} If the range is larger than half the heap.
4269
+ * @private
4270
+ */
4271
+ async _symbolAreaOf(field) {
4272
+ assert4__default.default(
4273
+ this._header && this._symbolTableOffset !== null,
4274
+ "The QVD file header has not been parsed before its symbols were read."
4275
+ );
4276
+ const area = this._symbolAreaPlan().byField.get(field);
4277
+ assert4__default.default(area, "A field this read did not select has no symbol area.");
4278
+ const { range } = area;
4279
+ if (range.buffer === null) {
4280
+ const length = range.end - range.start;
4281
+ validateSymbolTableSize(length, this._path, headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]));
4282
+ const buffer = Buffer.alloc(length);
4283
+ const from = this._symbolTableOffset + range.start;
4284
+ await this._readAt(buffer, 0, length, from, from + length);
4285
+ range.buffer = buffer;
4286
+ }
4287
+ return { buffer: range.buffer, start: area.start - range.start, end: area.end - range.start, base: range.start };
4288
+ }
4289
+ /**
4290
+ * Lets go of a field's symbols once they are parsed, and of the bytes of its range once every field in it
4291
+ * has been.
4292
+ *
4293
+ * Every text is copied out of the bytes as it is decoded, so what the read keeps is the values. A read of
4294
+ * one field of a file whose other fields are large therefore holds that field's bytes and no others.
4295
+ *
4296
+ * @param {any} field The field whose symbols are parsed.
4297
+ * @private
4298
+ */
4299
+ _releaseSymbolArea(field) {
4300
+ const area = this._symbolAreaPlan().byField.get(field);
4301
+ assert4__default.default(area, "A field this read did not select has no symbol area.");
4302
+ area.range.fields -= 1;
4303
+ if (area.range.fields === 0) {
4304
+ area.range.buffer = null;
4305
+ }
4306
+ }
4307
+ /**
4308
+ * Reads records from the open file a slice at a time, and hands each slice to `visit`.
4309
+ *
4310
+ * One buffer of at most `sliceBytes` holds a slice, and is reused for the next one, so a read of any
4311
+ * number of records holds that much of them and no more. Cancellation is checked before each slice.
4312
+ *
4313
+ * @param {number} firstRow The file row of the first record.
4314
+ * @param {number} rowCount How many records.
4315
+ * @param {number} recordSize Bytes per record.
4316
+ * @param {(slice: Buffer, done: number, count: number) => void|Promise<void>} visit Called with each
4317
+ * slice's records, how many records came before it, and how many it holds.
4318
+ * @private
4319
+ */
4320
+ async _forEachSlice(firstRow, rowCount, recordSize, visit) {
4321
+ if (rowCount === 0) {
4322
+ return;
4323
+ }
4324
+ assert4__default.default(this._indexTableOffset !== null, "The QVD file header has not been parsed before its records were read.");
4325
+ const sliceRows = this._sliceRowsFor(rowCount, recordSize);
4326
+ const slice = Buffer.alloc(sliceRows * recordSize);
4327
+ const requiredBytes = this._indexTableOffset + (firstRow + rowCount) * recordSize;
4328
+ for (let done = 0; done < rowCount; done += sliceRows) {
4329
+ this._throwIfAborted();
4330
+ const count = Math.min(sliceRows, rowCount - done);
4331
+ const records = slice.subarray(0, count * recordSize);
4332
+ await this._readAt(
4333
+ records,
4334
+ 0,
4335
+ records.length,
4336
+ this._indexTableOffset + (firstRow + done) * recordSize,
4337
+ requiredBytes
4338
+ );
4339
+ await visit(records, done, count);
4340
+ }
4341
+ }
3773
4342
  /**
3774
4343
  * Parses the XML header of the QVD file. This method is part of the parsing process
3775
4344
  * and should not be called directly.
3776
4345
  */
3777
4346
  async _parseHeader() {
3778
- if (!this._buffer) {
4347
+ if (!this._headerBuffer) {
3779
4348
  throw new exports.QvdCorruptedError(
3780
4349
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
3781
4350
  {
@@ -3786,7 +4355,7 @@ var init_QvdFileReader = __esm({
3786
4355
  }
3787
4356
  const HEADER_DELIMITER = "\r\n\0";
3788
4357
  const headerBeginIndex = 0;
3789
- const headerDelimiterIndex = this._buffer.indexOf(HEADER_DELIMITER, headerBeginIndex);
4358
+ const headerDelimiterIndex = this._headerBuffer.indexOf(HEADER_DELIMITER, headerBeginIndex);
3790
4359
  if (headerDelimiterIndex === -1) {
3791
4360
  throw new exports.QvdCorruptedError(
3792
4361
  "The XML header section does not exist or is not properly delimited from the binary data.",
@@ -3797,7 +4366,7 @@ var init_QvdFileReader = __esm({
3797
4366
  );
3798
4367
  }
3799
4368
  const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
3800
- const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
4369
+ const headerBuffer = this._headerBuffer.subarray(headerBeginIndex, headerEndIndex);
3801
4370
  this._fieldBitMetadataValidated = false;
3802
4371
  this._header = await parseHeaderXml(headerBuffer.toString(), this._path, "parseHeader");
3803
4372
  const fieldList = validateHeaderStructure(this._header, this._path, "parseHeader");
@@ -3819,12 +4388,12 @@ var init_QvdFileReader = __esm({
3819
4388
  * @param {QvdRowWindow} window The rows of interest, as file row indices.
3820
4389
  * @param {string} stage Stage name for any error raised here.
3821
4390
  * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
3822
- * indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
3823
- * window's first record, file row `firstRow`, so the decoder always counts from zero.
4391
+ * firstRow: number}} The record geometry: the window's `rowsToLoad` records start at file row
4392
+ * `firstRow`, and `_forEachSlice` reads them.
3824
4393
  * @private
3825
4394
  */
3826
4395
  _planIndexTable(window, stage) {
3827
- if (!this._buffer || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
4396
+ if (!this._handle || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
3828
4397
  throw new exports.QvdCorruptedError(
3829
4398
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
3830
4399
  {
@@ -3839,22 +4408,18 @@ var init_QvdFileReader = __esm({
3839
4408
  const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3840
4409
  const indexTableLength = headerInteger(this._header["QvdTableHeader"]["Length"]);
3841
4410
  const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
4411
+ assert4__default.default(this._fileSize !== null, "The QVD file has not been measured before its records were planned.");
3842
4412
  validateIndexTableMetadata(
3843
4413
  recordSize,
3844
4414
  totalRows,
3845
4415
  indexTableLength,
3846
4416
  this._indexTableOffset,
3847
- this._buffer.length,
4417
+ this._fileSize,
3848
4418
  rowsToLoad,
3849
4419
  this._path,
3850
4420
  this._fileSize,
3851
4421
  firstRow,
3852
- this._bufferFirstRow
3853
- );
3854
- const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
3855
- const indexBuffer = this._buffer.subarray(
3856
- this._indexTableOffset + bufferRecordStart,
3857
- this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
4422
+ 0
3858
4423
  );
3859
4424
  if (!this._fieldBitMetadataValidated) {
3860
4425
  for (const field of allFields) {
@@ -3863,11 +4428,12 @@ var init_QvdFileReader = __esm({
3863
4428
  validateBitFields(allFields, this._path);
3864
4429
  this._fieldBitMetadataValidated = true;
3865
4430
  }
3866
- assert3__default.default(
3867
- rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
3868
- `The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
4431
+ const windowEnd = this._indexTableOffset + (firstRow + rowsToLoad) * recordSize;
4432
+ assert4__default.default(
4433
+ rowsToLoad === 0 || recordSize === 0 || windowEnd <= this._fileSize,
4434
+ `The window's records end at byte ${windowEnd} of a file of ${this._fileSize}, but ${rowsToLoad} were validated as present.`
3869
4435
  );
3870
- return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
4436
+ return { fields, recordSize, totalRows, rowsToLoad, firstRow };
3871
4437
  }
3872
4438
  /**
3873
4439
  * Analyzes the index table to determine which symbols are actually needed.
@@ -3883,48 +4449,58 @@ var init_QvdFileReader = __esm({
3883
4449
  * @private
3884
4450
  */
3885
4451
  async _analyzeIndexTableSymbolUsage(window) {
3886
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
4452
+ const { fields, recordSize, rowsToLoad, firstRow } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
3887
4453
  const symbolUsage = [];
3888
4454
  const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
3889
4455
  const column = new Int32Array(sliceRows);
3890
- fields.forEach((field, position) => {
3891
- this._throwIfAborted();
4456
+ const state = fields.map((field) => {
3892
4457
  const needed = /* @__PURE__ */ new Set();
3893
- symbolUsage[position] = needed;
3894
- const bitOffset = headerInteger(field["BitOffset"]);
3895
- const bitWidth = headerInteger(field["BitWidth"]);
3896
- const bias = headerInteger(field["Bias"]);
4458
+ symbolUsage.push(needed);
3897
4459
  const length = headerInteger(field["Length"]);
3898
- let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
3899
- let counted = false;
3900
- for (let first = 0; first < rowsToLoad; first += sliceRows) {
3901
- const count = Math.min(sliceRows, rowsToLoad - first);
3902
- decodeIndexColumn(
3903
- first === 0 ? indexBuffer : indexBuffer.subarray(first * recordSize),
3904
- recordSize,
3905
- count,
3906
- bitOffset,
3907
- bitWidth,
3908
- bias,
3909
- column
3910
- );
3911
- for (let row = 0; row < count; row++) {
3912
- if (column[row] >= 0 && column[row] < indexLimit) {
3913
- needed.add(column[row]);
4460
+ return {
4461
+ field,
4462
+ needed,
4463
+ bitOffset: headerInteger(field["BitOffset"]),
4464
+ bitWidth: headerInteger(field["BitWidth"]),
4465
+ bias: headerInteger(field["Bias"]),
4466
+ indexLimit: Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity,
4467
+ counted: false
4468
+ };
4469
+ });
4470
+ await this._forEachSlice(firstRow, rowsToLoad, recordSize, async (records, done, recordCount) => {
4471
+ for (let first = 0; first < recordCount; first += sliceRows) {
4472
+ const count = Math.min(sliceRows, recordCount - first);
4473
+ for (const field of state) {
4474
+ decodeIndexColumn(
4475
+ first === 0 ? records : records.subarray(first * recordSize),
4476
+ recordSize,
4477
+ count,
4478
+ field.bitOffset,
4479
+ field.bitWidth,
4480
+ field.bias,
4481
+ column
4482
+ );
4483
+ for (let row = 0; row < count; row++) {
4484
+ if (column[row] >= 0 && column[row] < field.indexLimit) {
4485
+ field.needed.add(column[row]);
4486
+ }
3914
4487
  }
3915
- }
3916
- if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
3917
- counted = true;
3918
- indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
3919
- for (const index of needed) {
3920
- if (index >= indexLimit) {
3921
- needed.delete(index);
4488
+ if (!field.counted && field.needed.size > COUNT_SYMBOLS_PAST) {
4489
+ field.counted = true;
4490
+ field.indexLimit = Math.min(field.indexLimit, await this._countFieldSymbols(field.field));
4491
+ for (const index of field.needed) {
4492
+ if (index >= field.indexLimit) {
4493
+ field.needed.delete(index);
4494
+ }
3922
4495
  }
3923
4496
  }
3924
4497
  }
3925
4498
  }
3926
- this._emitProgress("symbol-analysis", position + 1, fields.length);
4499
+ this._emitProgress("symbol-analysis", done + recordCount, rowsToLoad);
3927
4500
  });
4501
+ if (rowsToLoad === 0) {
4502
+ this._emitProgress("symbol-analysis", 0, 0);
4503
+ }
3928
4504
  return symbolUsage;
3929
4505
  }
3930
4506
  /**
@@ -3932,28 +4508,27 @@ var init_QvdFileReader = __esm({
3932
4508
  *
3933
4509
  * The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
3934
4510
  * `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
3935
- * validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
3936
- * same way wherever it is met.
4511
+ * validated before it is read, by `_symbolAreaPlan`, so a damaged `Offset`, `Length` or `NoOfSymbols` is
4512
+ * reported the same way wherever it is met. Its bytes are kept for the parse that follows. The walk checks the count against `NoOfSymbols` as the
4513
+ * parse does, so a field whose count is wrong is refused here, before the pass keeps anything on the
4514
+ * strength of it.
3937
4515
  *
3938
4516
  * @param {any} field The field's header.
3939
- * @return {number} Its symbols.
3940
- * @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
4517
+ * @return {Promise<number>} Its symbols.
4518
+ * @throws {QvdCorruptedError} If the area is not inside the symbol table, a symbol runs past it, or it
4519
+ * holds a different number of symbols from its `NoOfSymbols`.
3941
4520
  * @private
3942
4521
  */
3943
- _countFieldSymbols(field) {
3944
- assert3__default.default(
3945
- this._buffer && this._symbolTableOffset && this._indexTableOffset,
3946
- "The QVD file has not been read before its symbols were counted."
3947
- );
3948
- const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3949
- validateFieldMetadata(field, symbolBuffer.length, this._path);
3950
- const offset = headerInteger(field["Offset"]);
4522
+ async _countFieldSymbols(field) {
4523
+ const area = await this._symbolAreaOf(field);
3951
4524
  return countFieldSymbols(
3952
- symbolBuffer,
3953
- offset,
3954
- offset + headerInteger(field["Length"]),
4525
+ area.buffer,
4526
+ area.start,
4527
+ area.end,
4528
+ headerInteger(field["NoOfSymbols"]),
3955
4529
  field["FieldName"],
3956
- this._path
4530
+ this._path,
4531
+ area.base
3957
4532
  );
3958
4533
  }
3959
4534
  /**
@@ -3973,7 +4548,7 @@ var init_QvdFileReader = __esm({
3973
4548
  * that is fewer than the window covers - see `_prepare`.
3974
4549
  */
3975
4550
  async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
3976
- if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
4551
+ if (!this._handle || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
3977
4552
  throw new exports.QvdCorruptedError(
3978
4553
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
3979
4554
  {
@@ -3984,44 +4559,58 @@ var init_QvdFileReader = __esm({
3984
4559
  }
3985
4560
  const allFields = this._allFields;
3986
4561
  const fields = this._selectedFields;
3987
- const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
3988
- const symbolTableSize = symbolBuffer.length;
4562
+ const symbolTableSize = this._symbolTableLength();
4563
+ const plan = this._symbolAreaPlan();
4564
+ const symbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
3989
4565
  const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
3990
- validateSymbolTableSize(symbolTableSize, this._path, totalRows);
4566
+ const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
4567
+ validateSymbolTableSize(symbolBytes, this._path, totalRows);
3991
4568
  if (this._headerMatchesFile) {
3992
4569
  validateMemoryAvailability(
3993
- symbolTableSize,
4570
+ symbolBytes,
3994
4571
  rowsToLoad,
3995
4572
  totalRows,
3996
4573
  this._path,
3997
4574
  this._memorySafetyFactor,
3998
4575
  fields.length,
3999
4576
  this._materialisesRows,
4000
- liveRows
4577
+ liveRows,
4578
+ this._bytesHeld(symbolBytes, rowsToLoad, recordSize, liveRows, false),
4579
+ // `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
4580
+ // run has read the window's records once already - so the read's total is two passes over them.
4581
+ symbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize
4001
4582
  );
4002
4583
  }
4003
- warnLargeSymbolTable(symbolTableSize, rowsToLoad, totalRows, fields.length, this._materialisesRows);
4584
+ warnLargeSymbolTable(symbolBytes, rowsToLoad, totalRows, fields.length, this._materialisesRows);
4004
4585
  for (const field of allFields) {
4005
- validateFieldMetadata(field, symbolBuffer.length, this._path);
4586
+ validateFieldMetadata(field, symbolTableSize, this._path);
4006
4587
  }
4007
4588
  validateSymbolAreas(allFields, this._path);
4008
- this._symbolTable = fields.map((field, position) => {
4589
+ const symbolTable = [];
4590
+ for (const [position, field] of fields.entries()) {
4009
4591
  this._throwIfAborted();
4010
- const symbolsOffset = headerInteger(field["Offset"]);
4011
- const symbolsLength = headerInteger(field["Length"]);
4012
- const symbols = parseFieldSymbols(
4013
- symbolBuffer,
4014
- symbolsOffset,
4015
- symbolsOffset + symbolsLength,
4016
- // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
4017
- // `this._selectedFields`, so position is the one key that cannot collide.
4018
- symbolsToKeep ? symbolsToKeep[position] : null,
4019
- field["FieldName"],
4020
- this._path
4592
+ const area = await this._symbolAreaOf(field);
4593
+ symbolTable.push(
4594
+ parseFieldSymbols(
4595
+ area.buffer,
4596
+ area.start,
4597
+ area.end,
4598
+ // Checked against the symbols the area holds, which is the one check that sees a terminator
4599
+ // damaged in the middle of it (#124).
4600
+ headerInteger(field["NoOfSymbols"]),
4601
+ // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
4602
+ // `this._selectedFields`, so position is the one key that cannot collide.
4603
+ symbolsToKeep ? symbolsToKeep[position] : null,
4604
+ field["FieldName"],
4605
+ this._path,
4606
+ void 0,
4607
+ area.base
4608
+ )
4021
4609
  );
4610
+ this._releaseSymbolArea(field);
4022
4611
  this._emitProgress("symbol-table", position + 1, fields.length);
4023
- return symbols;
4024
- });
4612
+ }
4613
+ this._symbolTable = symbolTable;
4025
4614
  }
4026
4615
  /**
4027
4616
  * Parses the bit stuffed index table of the QVD file. This method is part of the parsing process
@@ -4048,27 +4637,46 @@ var init_QvdFileReader = __esm({
4048
4637
  * straight to the caller. Rows outside the window are not decoded, so they are not checked.
4049
4638
  *
4050
4639
  * @param {QvdRowWindow} window The rows to decode.
4640
+ * @param {number} [progressBase=0] Rows decoded before this call, so that progress over a chunked
4641
+ * iteration counts the whole window rather than restarting at every chunk - what `_buildRows` takes
4642
+ * for the same reason.
4643
+ * @param {number|null} [progressTotal=null] Rows the whole window covers, or null for this call's own.
4644
+ * @param {Array<Int32Array>|null} [into=null] Arrays to decode into, one per selected field and at least
4645
+ * `limit` long, for a caller that decodes chunk after chunk and keeps none of them. Null allocates.
4051
4646
  * @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
4052
4647
  */
4053
- async _parseIndexTable(window) {
4054
- const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
4055
- assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
4648
+ async _parseIndexTable(window, progressBase = 0, progressTotal = null, into = null) {
4649
+ const { fields, recordSize, rowsToLoad, firstRow } = this._planIndexTable(window, "parseIndexTable");
4650
+ const decodedBefore = progressBase;
4651
+ const decodedTotal = progressTotal === null ? rowsToLoad : progressTotal;
4652
+ assert4__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
4056
4653
  const symbolTable = this._symbolTable;
4057
- const columns = fields.map((field, position) => {
4058
- this._throwIfAborted();
4059
- const column = decodeIndexColumn(
4060
- indexBuffer,
4061
- recordSize,
4062
- rowsToLoad,
4063
- headerInteger(field["BitOffset"]),
4064
- headerInteger(field["BitWidth"]),
4065
- headerInteger(field["Bias"]),
4066
- new Int32Array(rowsToLoad),
4067
- { symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
4068
- );
4069
- this._emitProgress("index-table", position + 1, fields.length);
4070
- return column;
4654
+ const decoders = fields.map((field, position) => ({
4655
+ bitOffset: headerInteger(field["BitOffset"]),
4656
+ bitWidth: headerInteger(field["BitWidth"]),
4657
+ bias: headerInteger(field["Bias"]),
4658
+ symbolCount: symbolTable[position].numbers.length,
4659
+ name: field["FieldName"]
4660
+ }));
4661
+ const columns = into === null ? fields.map(() => new Int32Array(rowsToLoad)) : into.map((codes) => codes.subarray(0, rowsToLoad));
4662
+ await this._forEachSlice(firstRow, rowsToLoad, recordSize, (records, done, count) => {
4663
+ decoders.forEach((decoder, position) => {
4664
+ decodeIndexColumn(
4665
+ records,
4666
+ recordSize,
4667
+ count,
4668
+ decoder.bitOffset,
4669
+ decoder.bitWidth,
4670
+ decoder.bias,
4671
+ columns[position].subarray(done, done + count),
4672
+ { symbolCount: decoder.symbolCount, field: decoder.name, file: this._path, firstRow: firstRow + done }
4673
+ );
4674
+ });
4675
+ this._emitProgress("index-table", decodedBefore + done + count, decodedTotal);
4071
4676
  });
4677
+ if (rowsToLoad === 0) {
4678
+ this._emitProgress("index-table", decodedBefore, decodedTotal);
4679
+ }
4072
4680
  this._indexColumns = columns;
4073
4681
  this._rowsDecoded = rowsToLoad;
4074
4682
  }
@@ -4087,31 +4695,122 @@ var init_QvdFileReader = __esm({
4087
4695
  * @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
4088
4696
  */
4089
4697
  async loadMetadata() {
4090
- await this._readData({ offset: 0, limit: null }, true);
4091
- this._emitProgress("header", 0, 1);
4092
- await this._parseHeader();
4093
- this._emitProgress("header", 1, 1);
4094
- this._throwIfAborted();
4095
- assert3__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
4096
- const header = this._header["QvdTableHeader"];
4097
- const columns = this._allFields.map((field) => field["FieldName"]);
4098
- const rowCount = headerInteger(header["NoOfRecords"]);
4099
- validateRecordCount(rowCount, this._path, "readMetadata");
4100
- const shape = new exports.QvdDataFrame([], columns, header, {
4101
- symbolTableBytes: headerInteger(header["Offset"]),
4102
- totalRows: rowCount,
4103
- rowsLoaded: 0,
4104
- symbolFiltering: false,
4105
- symbolsKept: null
4698
+ return await this._closingAfter(async () => {
4699
+ await this._readData({ offset: 0, limit: null }, true);
4700
+ this._emitProgress("header", 0, 1);
4701
+ await this._parseHeader();
4702
+ this._emitProgress("header", 1, 1);
4703
+ this._throwIfAborted();
4704
+ assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
4705
+ const header = this._header["QvdTableHeader"];
4706
+ const columns = this._allFields.map((field) => field["FieldName"]);
4707
+ const rowCount = headerInteger(header["NoOfRecords"]);
4708
+ validateRecordCount(rowCount, this._path, "readMetadata");
4709
+ const shape = new exports.QvdDataFrame([], columns, header, {
4710
+ symbolTableBytes: headerInteger(header["Offset"]),
4711
+ totalRows: rowCount,
4712
+ rowsLoaded: 0,
4713
+ symbolFiltering: false,
4714
+ symbolsKept: null
4715
+ });
4716
+ return {
4717
+ columns,
4718
+ rowCount,
4719
+ columnCount: columns.length,
4720
+ fields: columns.map((name) => shape.getFieldMetadata(name)),
4721
+ fileMetadata: shape.fileMetadata,
4722
+ metadata: header
4723
+ };
4724
+ });
4725
+ }
4726
+ /**
4727
+ * What a read of this file would cost, and whether it fits, without reading it.
4728
+ *
4729
+ * Reads the header and the file's size and nothing else, at the constant cost of `loadMetadata()`,
4730
+ * then asks the same question a read asks before it allocates anything - through the same function,
4731
+ * from the same numbers. That is the whole point: an answer computed a second way would be a second
4732
+ * opinion, and a read this approves would still be refused.
4733
+ *
4734
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [rawWindow]
4735
+ * The rows the read would cover, spelled any of the ways a read accepts.
4736
+ * @param {{chunkSize?: number|null}} [options] `chunkSize` when the read would be an `iterate()`,
4737
+ * which holds two chunks of rows rather than the window.
4738
+ * @return {Promise<any>} The answer - see `checkMemory`.
4739
+ */
4740
+ async checkRead(rawWindow, { chunkSize = null } = {}) {
4741
+ const window = normaliseWindow(rawWindow, this._path);
4742
+ return await this._closingAfter(async () => {
4743
+ await this._readData({ offset: 0, limit: null }, true);
4744
+ this._emitProgress("header", 0, 1);
4745
+ await this._parseHeader();
4746
+ this._emitProgress("header", 1, 1);
4747
+ this._throwIfAborted();
4748
+ assert4__default.default(
4749
+ this._header && this._selectedFields && this._allFields && this._symbolTableOffset !== null,
4750
+ "The QVD file header has not been parsed."
4751
+ );
4752
+ const header = this._header["QvdTableHeader"];
4753
+ const totalRows = headerInteger(header["NoOfRecords"]);
4754
+ const recordSize = headerInteger(header["RecordByteSize"]);
4755
+ const symbolTableLength = headerInteger(header["Offset"]);
4756
+ const selected = this._selectedFields;
4757
+ validateRecordSize(recordSize, this._path, "checkRead");
4758
+ validateRecordCount(totalRows, this._path, "checkRead");
4759
+ if (!this._headerMatchesFile) {
4760
+ throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
4761
+ file: this._path,
4762
+ fileSize: this._fileSize,
4763
+ requiredBytes: this._symbolTableOffset + symbolTableLength + totalRows * recordSize,
4764
+ stage: "checkRead"
4765
+ });
4766
+ }
4767
+ const tableLength = this._symbolTableLength();
4768
+ for (const field of this._allFields) {
4769
+ validateFieldMetadata(field, tableLength, this._path);
4770
+ validateFieldBitMetadata(field, recordSize, this._path);
4771
+ }
4772
+ validateSymbolAreas(this._allFields, this._path);
4773
+ const resolved = resolveWindow(window, totalRows);
4774
+ const windowRows = resolved.limit;
4775
+ const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
4776
+ const analysisAhead = this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
4777
+ const measured = getMemoryBudget();
4778
+ const ask = /* @__PURE__ */ __name((fields, rows) => {
4779
+ const bytes = symbolBytesOf(fields, symbolTableLength);
4780
+ return checkMemory({
4781
+ measured,
4782
+ symbolTableSize: bytes,
4783
+ maxRows: rows,
4784
+ totalRows,
4785
+ safetyFactor: this._memorySafetyFactor,
4786
+ columnCount: fields.length,
4787
+ materialisesRows: this._materialisesRows,
4788
+ live: liveRows,
4789
+ bytesHeld: this._bytesHeld(bytes, rows, recordSize, liveRows, analysisAhead),
4790
+ // What it reads from the file, which is not what it holds: the symbol areas, and every record
4791
+ // the window covers, read a slice at a time and not kept - twice over where the symbol-usage
4792
+ // pass will run, since it reads them before the decode reads them again.
4793
+ readBytes: bytes + readPasses(analysisAhead) * windowRows * recordSize
4794
+ });
4795
+ }, "ask");
4796
+ const answer = ask(selected, windowRows);
4797
+ if (!answer.fits && selected.length > 1) {
4798
+ const bySize = [...selected].sort(
4799
+ (a, b) => headerInteger(a["Length"]) - headerInteger(b["Length"])
4800
+ );
4801
+ for (let take = selected.length - 1; take >= 1; take -= 1) {
4802
+ const fewer = bySize.slice(0, take);
4803
+ if (ask(fewer, windowRows).fits) {
4804
+ answer.suggestions.push({
4805
+ option: "fields",
4806
+ value: fewer.map((field) => field["FieldName"])
4807
+ });
4808
+ break;
4809
+ }
4810
+ }
4811
+ }
4812
+ return answer;
4106
4813
  });
4107
- return {
4108
- columns,
4109
- rowCount,
4110
- columnCount: columns.length,
4111
- fields: columns.map((name) => shape.getFieldMetadata(name)),
4112
- fileMetadata: shape.fileMetadata,
4113
- metadata: header
4114
- };
4115
4814
  }
4116
4815
  /**
4117
4816
  * Loads the QVD file into memory and parses it.
@@ -4126,19 +4825,21 @@ var init_QvdFileReader = __esm({
4126
4825
  */
4127
4826
  async load(window = null) {
4128
4827
  const rows = normaliseWindow(window, this._path);
4129
- const prepared = await this._prepare(rows);
4130
- await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
4131
- const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
4132
- return new exports.QvdDataFrame(
4133
- data,
4134
- prepared.columns,
4135
- prepared.metadata,
4136
- {
4137
- ...prepared.loadStats,
4138
- rowsLoaded: data.length
4139
- },
4140
- prepared.storedSymbols
4141
- );
4828
+ return await this._closingAfter(async () => {
4829
+ const prepared = await this._prepare(rows);
4830
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
4831
+ const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
4832
+ return new exports.QvdDataFrame(
4833
+ data,
4834
+ prepared.columns,
4835
+ prepared.metadata,
4836
+ {
4837
+ ...prepared.loadStats,
4838
+ rowsLoaded: data.length
4839
+ },
4840
+ prepared.storedSymbols
4841
+ );
4842
+ });
4142
4843
  }
4143
4844
  /**
4144
4845
  * Reads the file as columns, without ever materialising rows.
@@ -4157,19 +4858,21 @@ var init_QvdFileReader = __esm({
4157
4858
  */
4158
4859
  async loadColumnar(window = null) {
4159
4860
  const rows = normaliseWindow(window, this._path);
4160
- const prepared = await this._prepare(rows, null, true);
4161
- await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
4162
- const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
4163
- assert3__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
4164
- return new QvdColumnTable2({
4165
- columns: prepared.columns,
4166
- codesByField: this._indexColumns,
4167
- symbolsByField: prepared.resolvedByField,
4168
- halvesByField: prepared.halvesByField,
4169
- rowCount: this._rowsDecoded,
4170
- metadata: prepared.metadata,
4171
- storedSymbols: prepared.storedSymbols,
4172
- loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
4861
+ return await this._closingAfter(async () => {
4862
+ const prepared = await this._prepare(rows, null, true);
4863
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
4864
+ const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
4865
+ assert4__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
4866
+ return new QvdColumnTable2({
4867
+ columns: prepared.columns,
4868
+ codesByField: this._indexColumns,
4869
+ symbolsByField: prepared.resolvedByField,
4870
+ halvesByField: prepared.halvesByField,
4871
+ rowCount: this._rowsDecoded,
4872
+ metadata: prepared.metadata,
4873
+ storedSymbols: prepared.storedSymbols,
4874
+ loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
4875
+ });
4173
4876
  });
4174
4877
  }
4175
4878
  /**
@@ -4198,37 +4901,41 @@ var init_QvdFileReader = __esm({
4198
4901
  * @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
4199
4902
  */
4200
4903
  async *iterateRows(window, chunkSize) {
4201
- if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
4202
- throw new exports.QvdValidationError("chunkSize must be a positive integer", {
4203
- provided: chunkSize,
4204
- type: typeof chunkSize,
4205
- file: this._path
4206
- });
4207
- }
4904
+ requireChunkSize(chunkSize, this._path);
4208
4905
  const liveRows = { rows: chunkSize * 2, perChunk: 2 };
4209
4906
  const rows = normaliseWindow(window, this._path);
4210
- const prepared = await this._prepare(rows, liveRows);
4211
- if (prepared.rowsAvailable === 0) {
4212
- this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
4213
- return;
4214
- }
4215
- for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
4216
- this._throwIfAborted();
4217
- const count = Math.min(chunkSize, prepared.rowsAvailable - done);
4218
- const offset = prepared.offset + done;
4219
- await this._parseIndexTable({ offset, limit: count });
4220
- const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
4221
- yield new exports.QvdDataFrame(
4222
- data,
4223
- prepared.columns,
4224
- prepared.metadata,
4225
- {
4226
- ...prepared.loadStats,
4227
- offset,
4228
- rowsLoaded: data.length
4229
- },
4230
- prepared.storedSymbols
4231
- );
4907
+ this._startRead();
4908
+ let failing = false;
4909
+ try {
4910
+ const prepared = await this._prepare(rows, liveRows);
4911
+ if (prepared.rowsAvailable === 0) {
4912
+ this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
4913
+ return;
4914
+ }
4915
+ const codes = prepared.columns.map(() => new Int32Array(Math.min(chunkSize, prepared.rowsAvailable)));
4916
+ for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
4917
+ this._throwIfAborted();
4918
+ const count = Math.min(chunkSize, prepared.rowsAvailable - done);
4919
+ const offset = prepared.offset + done;
4920
+ await this._parseIndexTable({ offset, limit: count }, done, prepared.rowsAvailable, codes);
4921
+ const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
4922
+ yield new exports.QvdDataFrame(
4923
+ data,
4924
+ prepared.columns,
4925
+ prepared.metadata,
4926
+ {
4927
+ ...prepared.loadStats,
4928
+ offset,
4929
+ rowsLoaded: data.length
4930
+ },
4931
+ prepared.storedSymbols
4932
+ );
4933
+ }
4934
+ } catch (error) {
4935
+ failing = true;
4936
+ throw error;
4937
+ } finally {
4938
+ await this._endRead(failing);
4232
4939
  }
4233
4940
  }
4234
4941
  /**
@@ -4262,23 +4969,21 @@ var init_QvdFileReader = __esm({
4262
4969
  await this._parseHeader();
4263
4970
  this._emitProgress("header", 1, 1);
4264
4971
  this._throwIfAborted();
4265
- assert3__default.default(this._header, "The QVD file header has not been parsed.");
4972
+ assert4__default.default(this._header, "The QVD file header has not been parsed.");
4266
4973
  const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
4267
4974
  const symbolTableLength = headerInteger(this._header["QvdTableHeader"]["Offset"]);
4268
4975
  const resolved = resolveWindow(window, totalRows);
4269
4976
  const rowsAvailable = resolved.limit;
4270
4977
  let symbolsToKeep = null;
4271
4978
  let symbolsKept = null;
4272
- if (window.limit !== null || window.offset > 0) {
4273
- if (symbolTableLength > this._symbolFilteringThreshold) {
4274
- symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
4275
- symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
4276
- }
4979
+ if (this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) {
4980
+ symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
4981
+ symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
4277
4982
  }
4278
4983
  await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
4279
- assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
4984
+ assert4__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
4280
4985
  this._throwIfAborted();
4281
- assert3__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
4986
+ assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
4282
4987
  const resolvedByField = [];
4283
4988
  const halvesByField = [];
4284
4989
  const entries = [];
@@ -4339,7 +5044,7 @@ var init_QvdFileReader = __esm({
4339
5044
  * @private
4340
5045
  */
4341
5046
  _buildRows(resolvedByField, progressBase, progressTotal) {
4342
- assert3__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
5047
+ assert4__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
4343
5048
  const indexColumns = this._indexColumns;
4344
5049
  const fieldCount = indexColumns.length;
4345
5050
  const rowCount = this._rowsDecoded;
@@ -5028,12 +5733,16 @@ var init_QvdDataFrame = __esm({
5028
5733
  /**
5029
5734
  * Reads a QVD file in chunks, as an async generator of data frames.
5030
5735
  *
5031
- * The file is opened, read and parsed once; only the index decode and the row building happen
5032
- * per chunk, so what this bounds is row materialisation - the part that actually dominates a
5033
- * large read's heap. It is **not** constant-memory reading of an arbitrarily large file: the
5034
- * symbol table is parsed in full whatever the chunk size, because a stored index in the last
5035
- * chunk can address the first symbol. On a high-cardinality file that table is the bulk of the
5036
- * cost, and `readMetadata` is the only read that avoids it.
5736
+ * The file is opened once and its symbol table parsed once. Each chunk's records are read from the
5737
+ * file when that chunk is built, so what this holds is the symbol table and two chunks of rows,
5738
+ * whatever the size of the file: a 20 GB QVD iterates in the memory its symbol table needs. That
5739
+ * table is still parsed in full whatever the chunk size, because a stored index in the last chunk
5740
+ * can address the first symbol. On a high-cardinality file that table is the bulk of the cost, and
5741
+ * `readMetadata` is the only read that avoids it.
5742
+ *
5743
+ * The file stays open until the iteration ends. Running it to the end closes it, and so do
5744
+ * `break` or a throw inside `for await` and a call to `return()` on the iterator; an iterator
5745
+ * abandoned part-way without any of those holds the file until it is garbage-collected.
5037
5746
  *
5038
5747
  * ```js
5039
5748
  * for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
@@ -5107,6 +5816,68 @@ var init_QvdDataFrame = __esm({
5107
5816
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
5108
5817
  return await new QvdFileReader2(path5, metadataOptionsFrom(options)).loadMetadata();
5109
5818
  }
5819
+ /**
5820
+ * Answers what a read would cost, and whether it fits, without doing it.
5821
+ *
5822
+ * Takes the options `fromQvd()` takes, plus `as` and `chunkSize` to say which read is being asked
5823
+ * about. Reads the header and the file's size and nothing else, at a cost that does not grow with
5824
+ * the file.
5825
+ *
5826
+ * The answer comes from the same function a read consults before it allocates anything, from the
5827
+ * same numbers, so **a read this approves is not refused later for memory** - and every suggestion
5828
+ * it carries has been read back through the check, so following one gives a read that fits.
5829
+ *
5830
+ * It answers about resources, so it answers only for a header it can trust. A header whose numbers
5831
+ * are not usable, or that claims more than the file holds, is refused as a `QvdCorruptedError` rather
5832
+ * than answered: sizing a read from numbers the file contradicts produced a memory verdict about a
5833
+ * file whose real problem was structural, and it was wrong in both directions - approving a read the
5834
+ * library then refused, and refusing another with advice that was refused too.
5835
+ *
5836
+ * That covers everything a reader can tell from the header: a field area past the end of the symbol
5837
+ * table, two fields claiming one area, a `Bias` that is neither 0 nor -2, a `BitWidth` past 31. Damage
5838
+ * that is not in the header - a value or an index the file has spoiled - is still found only by
5839
+ * reading, and still refused as a `QvdCorruptedError` after this has said the read fits.
5840
+ *
5841
+ * ```js
5842
+ * const answer = await QvdDataFrame.checkRead('huge.qvd', {as: 'columns', fields: ['Amount']});
5843
+ *
5844
+ * if (!answer.fits) {
5845
+ * console.log(answer.reason); // 'memory'
5846
+ * console.log(answer.suggestions); // [{option: 'limit', value: 1250000}, ...]
5847
+ * }
5848
+ * ```
5849
+ *
5850
+ * @param {string} path The QVD file.
5851
+ * @param {object} [options] What `fromQvd()` takes, plus the two below.
5852
+ * @param {'rows'|'columns'} [options.as='rows'] Which read is being asked about: `rows` builds row
5853
+ * arrays and `columns` does not, which is most of what a read costs.
5854
+ * @param {number|null} [options.chunkSize=null] The chunk an `iterate()` would use, which holds two
5855
+ * chunks of rows rather than the whole window.
5856
+ * @return {Promise<any>} The answer: `fits`, `reason` when it does not, `estimate`, `budget`,
5857
+ * `exact` and `suggestions`.
5858
+ * @throws {QvdValidationError} If an option's value is not valid, with `context.reason` of `option`.
5859
+ * @throws {QvdCorruptedError} If the header cannot be read, its numbers are not usable, or it claims
5860
+ * more than the file holds. The read refuses such a file too, though it may name the fault
5861
+ * differently - it gets there by planning the index table, where this gets there from the size.
5862
+ */
5863
+ static async checkRead(path5, options = {}) {
5864
+ const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
5865
+ const { as = "rows", chunkSize = null } = options;
5866
+ if (as !== "rows" && as !== "columns") {
5867
+ throw new exports.QvdValidationError("as must be 'rows' or 'columns'", {
5868
+ provided: as,
5869
+ reason: "option",
5870
+ option: "as",
5871
+ value: as,
5872
+ file: path5
5873
+ });
5874
+ }
5875
+ if (chunkSize !== null) {
5876
+ requireChunkSize(chunkSize, path5);
5877
+ }
5878
+ const reader = new QvdFileReader2(path5, { ...readerOptionsFrom(options), materialisesRows: as === "rows" });
5879
+ return await reader.checkRead(windowFrom(options), { chunkSize });
5880
+ }
5110
5881
  /**
5111
5882
  * Constructs a data frame from a dictionary.
5112
5883
  *