qvdjs 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -2197,13 +2197,13 @@ function estimateRowMemory(rows, columnCount) {
2197
2197
  }
2198
2198
  return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
2199
2199
  }
2200
- function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
2200
+ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null, wholeSymbols = false) {
2201
2201
  const FULL_PARSE_OVERHEAD = 6;
2202
2202
  const MINIMAL_OVERHEAD = 0.01;
2203
2203
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
2204
2204
  const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
2205
2205
  const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
2206
- if (maxRows === null || maxRows >= totalRows) {
2206
+ if (maxRows === null || maxRows >= totalRows || wholeSymbols) {
2207
2207
  return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
2208
2208
  }
2209
2209
  const rowPercentage = maxRows / totalRows;
@@ -2212,8 +2212,8 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
2212
2212
  const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
2213
2213
  return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
2214
2214
  }
2215
- function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
2216
- const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
2215
+ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false, wholeSymbols = false) {
2216
+ const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows, null, wholeSymbols) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
2217
2217
  if (costOf(totalRows) <= budget) {
2218
2218
  return totalRows;
2219
2219
  }
@@ -2232,11 +2232,19 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
2232
2232
  }
2233
2233
  return low;
2234
2234
  }
2235
- function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
2235
+ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false, wholeSymbols = false) {
2236
2236
  const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
2237
2237
  const fits = /* @__PURE__ */ __name((chunk) => {
2238
2238
  const live = Math.min(chunk * liveRowsPerChunk, covered);
2239
- const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
2239
+ const cost = estimateMemoryUsage(
2240
+ symbolTableSize,
2241
+ windowRows,
2242
+ totalRows,
2243
+ columnCount,
2244
+ true,
2245
+ chunk * liveRowsPerChunk,
2246
+ wholeSymbols
2247
+ ) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
2240
2248
  return cost <= budget;
2241
2249
  }, "fits");
2242
2250
  if (fits(covered)) {
@@ -2257,8 +2265,9 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
2257
2265
  }
2258
2266
  return low;
2259
2267
  }
2260
- function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null) {
2268
+ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false) {
2261
2269
  const answer = checkMemory({
2270
+ wholeSymbols,
2262
2271
  symbolTableSize,
2263
2272
  maxRows,
2264
2273
  totalRows,
@@ -2290,7 +2299,8 @@ function checkMemory({
2290
2299
  live = null,
2291
2300
  bytesHeld = null,
2292
2301
  readBytes = null,
2293
- measured = null
2302
+ measured = null,
2303
+ wholeSymbols = false
2294
2304
  }) {
2295
2305
  if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
2296
2306
  throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
@@ -2305,7 +2315,15 @@ function checkMemory({
2305
2315
  const rowsLive = live === null ? null : live.rows;
2306
2316
  const liveRowsPerChunk = live === null ? 1 : live.perChunk;
2307
2317
  const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
2308
- const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
2318
+ const heapMemory = estimateMemoryUsage(
2319
+ symbolTableSize,
2320
+ maxRows,
2321
+ totalRows,
2322
+ columnCount,
2323
+ materialisesRows,
2324
+ rowsLive,
2325
+ wholeSymbols
2326
+ );
2309
2327
  const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
2310
2328
  const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
2311
2329
  const bounded = budget.candidates.map((candidate) => {
@@ -2351,7 +2369,8 @@ function checkMemory({
2351
2369
  totalRows,
2352
2370
  columnCount,
2353
2371
  materialisesRows,
2354
- includeExternal
2372
+ includeExternal,
2373
+ wholeSymbols
2355
2374
  ), "fitting");
2356
2375
  const firstGuess = fitting(liveRows);
2357
2376
  const over = fitting(firstGuess);
@@ -2377,7 +2396,8 @@ function checkMemory({
2377
2396
  totalRows,
2378
2397
  columnCount,
2379
2398
  liveRowsPerChunk,
2380
- includeExternal
2399
+ includeExternal,
2400
+ wholeSymbols
2381
2401
  ), "chunkFitting");
2382
2402
  const callersChunk = chunked ? Math.max(1, Math.floor(rowsLive / Math.max(1, liveRowsPerChunk))) : 0;
2383
2403
  const firstChunk = chunked ? chunkFitting(callersChunk) : 0;
@@ -2467,13 +2487,21 @@ function budgetOf(budget, tightest, safetyFactor) {
2467
2487
  function formatCount(value) {
2468
2488
  return value.toLocaleString("en-US");
2469
2489
  }
2470
- function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
2490
+ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, wholeSymbols = false) {
2471
2491
  const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
2472
2492
  if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
2473
2493
  return;
2474
2494
  }
2475
2495
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
2476
- const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
2496
+ const estimatedMemory = estimateMemoryUsage(
2497
+ symbolTableSize,
2498
+ maxRows,
2499
+ totalRows,
2500
+ columnCount,
2501
+ materialisesRows,
2502
+ null,
2503
+ wholeSymbols
2504
+ );
2477
2505
  if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
2478
2506
  return;
2479
2507
  }
@@ -3601,6 +3629,26 @@ function symbolBytesOf(selected, symbolTableLength) {
3601
3629
  function readPasses(analysisAhead) {
3602
3630
  return analysisAhead ? 2 : 1;
3603
3631
  }
3632
+ function validateWatchers(onProgress, signal, path5) {
3633
+ if (onProgress !== void 0 && typeof onProgress !== "function") {
3634
+ throw new exports.QvdValidationError("onProgress must be a function", {
3635
+ provided: onProgress,
3636
+ type: typeof onProgress,
3637
+ reason: "option",
3638
+ option: "onProgress",
3639
+ file: path5
3640
+ });
3641
+ }
3642
+ if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
3643
+ throw new exports.QvdValidationError("signal must be an AbortSignal", {
3644
+ provided: signal,
3645
+ type: typeof signal,
3646
+ reason: "option",
3647
+ option: "signal",
3648
+ file: path5
3649
+ });
3650
+ }
3651
+ }
3604
3652
  var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, SLICE_BYTES, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
3605
3653
  var init_QvdFileReader = __esm({
3606
3654
  "src/QvdFileReader.js"() {
@@ -3625,6 +3673,7 @@ var init_QvdFileReader = __esm({
3625
3673
  __name(parseHeaderXml, "parseHeaderXml");
3626
3674
  __name(symbolBytesOf, "symbolBytesOf");
3627
3675
  __name(readPasses, "readPasses");
3676
+ __name(validateWatchers, "validateWatchers");
3628
3677
  exports.QvdFileReader = class {
3629
3678
  static {
3630
3679
  __name(this, "QvdFileReader");
@@ -3705,20 +3754,7 @@ var init_QvdFileReader = __esm({
3705
3754
  });
3706
3755
  }
3707
3756
  this._sliceBytes = sliceBytes;
3708
- if (onProgress !== void 0 && typeof onProgress !== "function") {
3709
- throw new exports.QvdValidationError("onProgress must be a function", {
3710
- provided: onProgress,
3711
- type: typeof onProgress,
3712
- file: this._path
3713
- });
3714
- }
3715
- if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
3716
- throw new exports.QvdValidationError("signal must be an AbortSignal", {
3717
- provided: signal,
3718
- type: typeof signal,
3719
- file: this._path
3720
- });
3721
- }
3757
+ validateWatchers(onProgress, signal, this._path);
3722
3758
  this._requestedFields = fields === void 0 ? null : fields;
3723
3759
  this._onProgress = onProgress;
3724
3760
  this._signal = signal;
@@ -3727,6 +3763,8 @@ var init_QvdFileReader = __esm({
3727
3763
  this._failed = null;
3728
3764
  this._reading = false;
3729
3765
  this._symbolAreas = null;
3766
+ this._symbolCache = null;
3767
+ this._cachedFor = null;
3730
3768
  this._headerOffset = null;
3731
3769
  this._symbolTableOffset = null;
3732
3770
  this._indexTableOffset = null;
@@ -3738,6 +3776,7 @@ var init_QvdFileReader = __esm({
3738
3776
  this._indexColumns = null;
3739
3777
  this._rowsDecoded = 0;
3740
3778
  this._fileSize = null;
3779
+ this._fileIdentity = null;
3741
3780
  this._headerMatchesFile = false;
3742
3781
  }
3743
3782
  /**
@@ -3966,8 +4005,9 @@ var init_QvdFileReader = __esm({
3966
4005
  const indexTableOffset = symbolTableOffset + symbolTableLength;
3967
4006
  const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
3968
4007
  const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
3969
- const { size: fileSize } = await handle.stat().catch(failed);
4008
+ const { size: fileSize, ino, dev, mtimeMs } = await handle.stat().catch(failed);
3970
4009
  this._fileSize = fileSize;
4010
+ this._fileIdentity = `${dev}:${ino}:${mtimeMs}:${fileSize}`;
3971
4011
  this._headerMatchesFile = false;
3972
4012
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
3973
4013
  (value) => Number.isSafeInteger(value) && value >= 0
@@ -3983,7 +4023,8 @@ var init_QvdFileReader = __esm({
3983
4023
  this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
3984
4024
  const selected = selectFields(headerFields, this._requestedFields, this._path);
3985
4025
  const columnCount = selected.length;
3986
- const symbolBytes = symbolBytesOf(selected, symbolTableLength);
4026
+ const symbolBytes = symbolBytesOf(this._fieldsHeldAfter(selected, headerFields), symbolTableLength);
4027
+ const readSymbolBytes = symbolBytesOf(this._fieldsReadBy(selected), symbolTableLength);
3987
4028
  const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
3988
4029
  const windowRows = resolved.limit;
3989
4030
  if (headerNumbersUsable && this._headerMatchesFile) {
@@ -3997,11 +4038,11 @@ var init_QvdFileReader = __esm({
3997
4038
  this._materialisesRows,
3998
4039
  liveRows,
3999
4040
  this._bytesHeld(
4000
- symbolBytes,
4041
+ readSymbolBytes,
4001
4042
  windowRows,
4002
4043
  recordSize,
4003
4044
  liveRows,
4004
- this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)
4045
+ this._analysisAhead(window, resolved, totalRows, symbolTableLength)
4005
4046
  ),
4006
4047
  // What it reads, which is not what it holds - the records go through one buffer and are not
4007
4048
  // kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
@@ -4009,7 +4050,11 @@ var init_QvdFileReader = __esm({
4009
4050
  // Twice over where the symbol-usage pass is still ahead of it: that pass reads the window's
4010
4051
  // records to find which symbols the rows use, and the decode then reads them again. Counted
4011
4052
  // once, the figure understated the I/O of exactly the reads that do the most of it.
4012
- symbolBytes + readPasses(this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) * windowRows * recordSize
4053
+ readSymbolBytes + readPasses(this._analysisAhead(window, resolved, totalRows, symbolTableLength)) * windowRows * recordSize,
4054
+ // A paging read keeps whole columns, so the estimate must not discount its symbols as a window's
4055
+ // sample of them - see `estimateMemoryUsage`. Under-charging is the direction that ends in a
4056
+ // heap-limit abort rather than an error.
4057
+ this._symbolCache !== null
4013
4058
  );
4014
4059
  }
4015
4060
  if (window.offset === 0 && window.limit === null) {
@@ -4194,6 +4239,191 @@ var init_QvdFileReader = __esm({
4194
4239
  const chunkRows = liveRows === null ? windowRows : Math.max(1, Math.floor(liveRows.rows / Math.max(1, liveRows.perChunk)));
4195
4240
  return analysisAhead ? Math.max(windowRows, chunkRows) : chunkRows;
4196
4241
  }
4242
+ /**
4243
+ * The fields this reader will be holding the symbols of once this read has finished.
4244
+ *
4245
+ * The ones it selects, and - while paging - the ones it decoded for an earlier page and kept. That
4246
+ * union is what the memory checks have to be sized by, because it is what is live: a reader four
4247
+ * pages into a wide file holds four columns' values whether or not this page asks about them, and a
4248
+ * check sized by this page alone would approve a fifth column that does not fit beside them.
4249
+ *
4250
+ * The same set for every check, so the ceilings, the guard and the pre-flight cannot disagree about
4251
+ * what a paging read costs. Without a cache it is just the selection, which is what every one-shot
4252
+ * read has always been sized by.
4253
+ *
4254
+ * @param {Array<any>} selected The fields this read selects.
4255
+ * @param {Array<any>} all Every field in the header, to find a cached one by name.
4256
+ * @return {Array<any>} The fields whose symbols will be live.
4257
+ * @private
4258
+ */
4259
+ _fieldsHeldAfter(selected, all) {
4260
+ if (this._symbolCache === null || this._symbolCache.size === 0) {
4261
+ return selected;
4262
+ }
4263
+ const names = new Set(selected.map((field) => field["FieldName"]));
4264
+ const cached = all.filter(
4265
+ (field) => !names.has(field["FieldName"]) && this._symbolCache !== null && this._symbolCache.has(field["FieldName"])
4266
+ );
4267
+ return [...selected, ...cached];
4268
+ }
4269
+ /**
4270
+ * The fields whose symbol areas this read will actually read.
4271
+ *
4272
+ * The selection, less anything already decoded and kept. `_fieldsHeldAfter` answers what the read will
4273
+ * be *holding*, which is the right figure for the heap; this is the right one for the bytes it buffers
4274
+ * while parsing and for the I/O it reports, because a cached column's area is left out of the plan
4275
+ * entirely and never read.
4276
+ *
4277
+ * Sized by the wrong one of the two, a warm page was charged external bytes for buffers it never
4278
+ * allocates - and external bytes bind against a container limit, so a page that fits could be refused -
4279
+ * and `estimate.readBytes` claimed I/O it does not perform: on four columns with three cached it
4280
+ * reported 3,155,600 bytes for a read of 788,930.
4281
+ *
4282
+ * @param {Array<any>} selected The fields this read selects.
4283
+ * @return {Array<any>} The fields whose areas will be read.
4284
+ * @private
4285
+ */
4286
+ _fieldsReadBy(selected) {
4287
+ if (this._symbolCache === null || this._symbolCache.size === 0) {
4288
+ return selected;
4289
+ }
4290
+ return selected.filter(
4291
+ (field) => this._symbolCache !== null && !this._symbolCache.has(field["FieldName"])
4292
+ );
4293
+ }
4294
+ /**
4295
+ * Keeps what this reader decodes, so that a later read of the same file does not decode it again.
4296
+ *
4297
+ * For a caller reading one file many times over - a `QvdFile` and its pages - and off by default,
4298
+ * because every other entry point is one read and would only be holding values nobody will ask for
4299
+ * again. Decoding the symbols is 72% of a page of a hundred rows from a 300,000-row file; the rest is
4300
+ * the open, the header, the records and the rows.
4301
+ *
4302
+ * It turns the two-pass symbol path off with it. That path decodes only the symbols a window's rows
4303
+ * use, which is right for one read and wrong for a cache: a later page asking for a row that uses a
4304
+ * skipped symbol would read `undefined` where the value is. So a cached field is always a whole
4305
+ * field, walked and checked against its `NoOfSymbols` like any other.
4306
+ *
4307
+ * @return {void}
4308
+ */
4309
+ beginPaging() {
4310
+ this._symbolCache = /* @__PURE__ */ new Map();
4311
+ }
4312
+ /**
4313
+ * Reads with the fields the caller names for this read alone, rather than the reader's own.
4314
+ *
4315
+ * A `QvdFile` is opened once and its pages may each name a projection, so the selection cannot be
4316
+ * fixed at construction as it is for every other entry point.
4317
+ *
4318
+ * It holds until the next call replaces it rather than being cleared by the read, so **every caller
4319
+ * sets it before every read**, passing the file's own fields where the page named none. A caller that
4320
+ * relied on it being empty would instead get the projection of whatever ran last: that is what made a
4321
+ * `check()` naming no fields answer for the previous page's columns.
4322
+ *
4323
+ * @param {Array<string>|null|undefined} fields The fields, or undefined to use the reader's own.
4324
+ * @return {void}
4325
+ */
4326
+ selectForNextRead(fields) {
4327
+ if (fields !== void 0) {
4328
+ this._requestedFields = fields;
4329
+ }
4330
+ }
4331
+ /**
4332
+ * Watches the next read with the caller's `onProgress` and `signal`, rather than the reader's own.
4333
+ *
4334
+ * Both belong to one call, and a reader is told them when it is built - so a `QvdFile` page that named
4335
+ * either used to get a reader of its own. That made passing a progress callback change what the read
4336
+ * did rather than only observing it: a fresh reader is not paging, so it took the two-pass symbol
4337
+ * path, reported a different `loadStats.symbolFiltering`, and cached nothing. An observer must not
4338
+ * change what it observes, and a caller must not have to choose between cancelling a page and paging
4339
+ * cheaply.
4340
+ *
4341
+ * Like `selectForNextRead`, it holds until the next call replaces it rather than being cleared by the
4342
+ * read, so a caller that sets it for one page and not the next is still watched on the next - pass the
4343
+ * file's own watchers explicitly, as `QvdFile._page` does, rather than leaving them out.
4344
+ *
4345
+ * @param {{onProgress?: Function, signal?: AbortSignal}} [watchers] What this read is watched with.
4346
+ * @return {void}
4347
+ */
4348
+ observeNextRead({ onProgress, signal } = {}) {
4349
+ validateWatchers(onProgress, signal, this._path);
4350
+ this._onProgress = onProgress;
4351
+ this._signal = signal;
4352
+ }
4353
+ /**
4354
+ * Whether the symbol-usage pass runs for this read, cache and all.
4355
+ *
4356
+ * `_analysisWouldRun` answers whether the window wants the pass; a paging read never takes it, because
4357
+ * a column decoded in part cannot be kept. Asked in one place because it was asked in two and they
4358
+ * disagreed: the pass was gated on the cache while the memory charge and `estimate.readBytes` were
4359
+ * not, so every page of a file above the threshold was charged a slice of records it never buffered
4360
+ * and reported twice the bytes it read.
4361
+ *
4362
+ * @param {QvdRowWindow} window The window as the caller spelled it.
4363
+ * @param {{offset: number, limit: number}} resolved Where it lands in this file.
4364
+ * @param {number} totalRows Rows the file declares.
4365
+ * @param {number} symbolTableLength The symbol table's declared length.
4366
+ * @return {boolean} Whether the pass will run.
4367
+ * @private
4368
+ */
4369
+ _analysisAhead(window, resolved, totalRows, symbolTableLength) {
4370
+ return this._symbolCache === null && this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
4371
+ }
4372
+ /**
4373
+ * Empties the cache when the header in front of us is not the one it was decoded from.
4374
+ *
4375
+ * The fingerprint is what a rewrite moves: the file's identity on disk - device, inode, modification
4376
+ * time and size - and then the header numbers, down to each cached field's own offset, length and
4377
+ * symbol count.
4378
+ *
4379
+ * The header numbers alone were not enough, and the gap is not exotic. `QvdFileWriter` carries
4380
+ * `CreateUtcTime` over from the metadata it is handed, so reading a QVD, changing one text to another
4381
+ * of the same byte length and writing it back leaves `CreateUtcTime`, `NoOfRecords`, `Offset` and
4382
+ * every field's `Offset`, `Length` and `NoOfSymbols` exactly as they were - a different file the
4383
+ * fingerprint could not tell from the first. The filesystem sees it either way: an atomic write
4384
+ * renames a new file into place, which changes the inode, and an in-place one moves `mtimeMs`.
4385
+ *
4386
+ * @return {void}
4387
+ * @private
4388
+ */
4389
+ _forgetCacheIfFileChanged() {
4390
+ if (this._symbolCache === null || this._symbolCache.size === 0) {
4391
+ return;
4392
+ }
4393
+ if (this._cachedFor !== this._fileFingerprint()) {
4394
+ this._symbolCache = /* @__PURE__ */ new Map();
4395
+ this._cachedFor = null;
4396
+ }
4397
+ }
4398
+ /**
4399
+ * What identifies the file this reader's cache was decoded from.
4400
+ *
4401
+ * @return {string} The fingerprint.
4402
+ * @private
4403
+ */
4404
+ _fileFingerprint() {
4405
+ assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
4406
+ const header = this._header["QvdTableHeader"];
4407
+ return [
4408
+ // First, because it is the only part that moves when a rewrite preserves the header's numbers.
4409
+ this._fileIdentity,
4410
+ header["CreateUtcTime"],
4411
+ header["NoOfRecords"],
4412
+ header["Offset"],
4413
+ ...this._allFields.map(
4414
+ (field) => `${field["FieldName"]}:${field["Offset"]}:${field["Length"]}:${field["NoOfSymbols"]}`
4415
+ )
4416
+ ].join("|");
4417
+ }
4418
+ /**
4419
+ * Drops everything this reader has decoded, so that nothing outlives the caller that wanted it.
4420
+ *
4421
+ * @return {void}
4422
+ */
4423
+ endPaging() {
4424
+ this._symbolCache = null;
4425
+ this._cachedFor = null;
4426
+ }
4197
4427
  /**
4198
4428
  * The symbol table's length, as much of it as the file holds: what the header declares, cut short where
4199
4429
  * the file ends. Known before a byte of the table is read, so everything that can refuse the table is
@@ -4222,7 +4452,13 @@ var init_QvdFileReader = __esm({
4222
4452
  * order, so ranges that touch are merged: a read of every field is one range, and so is a read of fields
4223
4453
  * that happen to be neighbours. A read of one field of twenty reads that field's area alone.
4224
4454
  *
4225
- * Built once per read, from the fields the read selected, and each field's metadata is checked as it is
4455
+ * A field whose symbols this reader already holds is left out, because its bytes are not wanted: the
4456
+ * ranges are what gets read, and including a cached field's span had a page read every byte of every
4457
+ * column it named, cached or not. Measured on four columns of 20,000 distinct texts, a page naming all
4458
+ * four with three of them cached read all four columns' bytes - 1,155,600 of them, where 288,930 were
4459
+ * needed. The decode was saved and the I/O was not, which on the files #122 is about is the whole cost.
4460
+ *
4461
+ * Built once per read, from the fields the read must read, and each field's metadata is checked as it is
4226
4462
  * added - a range is arithmetic on `Offset` and `Length`, and those have to be inside the table first.
4227
4463
  * `_parseSymbolTable` checks every field of the file, selected or not, before it parses any.
4228
4464
  *
@@ -4237,7 +4473,10 @@ var init_QvdFileReader = __esm({
4237
4473
  }
4238
4474
  assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved before their symbols were read.");
4239
4475
  const tableLength = this._symbolTableLength();
4240
- const areas = this._selectedFields.map((field) => {
4476
+ const toRead = this._selectedFields.filter(
4477
+ (field) => this._symbolCache === null || !this._symbolCache.has(field["FieldName"])
4478
+ );
4479
+ const areas = toRead.map((field) => {
4241
4480
  validateFieldMetadata(field, tableLength, this._path);
4242
4481
  const start = headerInteger(field["Offset"]);
4243
4482
  return { field, start, end: start + headerInteger(field["Length"]) };
@@ -4559,9 +4798,11 @@ var init_QvdFileReader = __esm({
4559
4798
  }
4560
4799
  const allFields = this._allFields;
4561
4800
  const fields = this._selectedFields;
4801
+ this._forgetCacheIfFileChanged();
4562
4802
  const symbolTableSize = this._symbolTableLength();
4563
4803
  const plan = this._symbolAreaPlan();
4564
- const symbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
4804
+ const readSymbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
4805
+ const symbolBytes = readSymbolBytes + symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
4565
4806
  const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
4566
4807
  const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
4567
4808
  validateSymbolTableSize(symbolBytes, this._path, totalRows);
@@ -4575,13 +4816,21 @@ var init_QvdFileReader = __esm({
4575
4816
  fields.length,
4576
4817
  this._materialisesRows,
4577
4818
  liveRows,
4578
- this._bytesHeld(symbolBytes, rowsToLoad, recordSize, liveRows, false),
4819
+ this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false),
4579
4820
  // `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
4580
4821
  // run has read the window's records once already - so the read's total is two passes over them.
4581
- symbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize
4822
+ readSymbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize,
4823
+ this._symbolCache !== null
4582
4824
  );
4583
4825
  }
4584
- warnLargeSymbolTable(symbolBytes, rowsToLoad, totalRows, fields.length, this._materialisesRows);
4826
+ warnLargeSymbolTable(
4827
+ symbolBytes,
4828
+ rowsToLoad,
4829
+ totalRows,
4830
+ fields.length,
4831
+ this._materialisesRows,
4832
+ this._symbolCache !== null
4833
+ );
4585
4834
  for (const field of allFields) {
4586
4835
  validateFieldMetadata(field, symbolTableSize, this._path);
4587
4836
  }
@@ -4589,24 +4838,36 @@ var init_QvdFileReader = __esm({
4589
4838
  const symbolTable = [];
4590
4839
  for (const [position, field] of fields.entries()) {
4591
4840
  this._throwIfAborted();
4841
+ const cached = this._symbolCache?.get(field["FieldName"]);
4842
+ if (cached) {
4843
+ symbolTable.push(cached);
4844
+ this._emitProgress("symbol-table", position + 1, fields.length);
4845
+ continue;
4846
+ }
4592
4847
  const area = await this._symbolAreaOf(field);
4593
- symbolTable.push(
4594
- parseFieldSymbols(
4595
- area.buffer,
4596
- area.start,
4597
- area.end,
4598
- // Checked against the symbols the area holds, which is the one check that sees a terminator
4599
- // damaged in the middle of it (#124).
4600
- headerInteger(field["NoOfSymbols"]),
4601
- // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
4602
- // `this._selectedFields`, so position is the one key that cannot collide.
4603
- symbolsToKeep ? symbolsToKeep[position] : null,
4604
- field["FieldName"],
4605
- this._path,
4606
- void 0,
4607
- area.base
4608
- )
4848
+ const parsed = parseFieldSymbols(
4849
+ area.buffer,
4850
+ area.start,
4851
+ area.end,
4852
+ // Checked against the symbols the area holds, which is the one check that sees a terminator
4853
+ // damaged in the middle of it (#124). A cached field was checked when it was decoded, which is
4854
+ // why the cache may only hold a field a full walk produced.
4855
+ headerInteger(field["NoOfSymbols"]),
4856
+ // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
4857
+ // `this._selectedFields`, so position is the one key that cannot collide.
4858
+ symbolsToKeep ? symbolsToKeep[position] : null,
4859
+ field["FieldName"],
4860
+ this._path,
4861
+ void 0,
4862
+ area.base
4609
4863
  );
4864
+ symbolTable.push(parsed);
4865
+ if (this._symbolCache && symbolsToKeep === null) {
4866
+ if (this._symbolCache.size === 0) {
4867
+ this._cachedFor = this._fileFingerprint();
4868
+ }
4869
+ this._symbolCache.set(field["FieldName"], parsed);
4870
+ }
4610
4871
  this._releaseSymbolArea(field);
4611
4872
  this._emitProgress("symbol-table", position + 1, fields.length);
4612
4873
  }
@@ -4701,28 +4962,39 @@ var init_QvdFileReader = __esm({
4701
4962
  await this._parseHeader();
4702
4963
  this._emitProgress("header", 1, 1);
4703
4964
  this._throwIfAborted();
4704
- assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
4705
- const header = this._header["QvdTableHeader"];
4706
- const columns = this._allFields.map((field) => field["FieldName"]);
4707
- const rowCount = headerInteger(header["NoOfRecords"]);
4708
- validateRecordCount(rowCount, this._path, "readMetadata");
4709
- const shape = new exports.QvdDataFrame([], columns, header, {
4710
- symbolTableBytes: headerInteger(header["Offset"]),
4711
- totalRows: rowCount,
4712
- rowsLoaded: 0,
4713
- symbolFiltering: false,
4714
- symbolsKept: null
4715
- });
4716
- return {
4717
- columns,
4718
- rowCount,
4719
- columnCount: columns.length,
4720
- fields: columns.map((name) => shape.getFieldMetadata(name)),
4721
- fileMetadata: shape.fileMetadata,
4722
- metadata: header
4723
- };
4965
+ return this.describeParsed();
4724
4966
  });
4725
4967
  }
4968
+ /**
4969
+ * The schema and header of the file this reader has parsed, as `readMetadata()` reports them.
4970
+ *
4971
+ * Built from the parsed header and nothing else, so a caller holding a header - a `QvdFile` - can have
4972
+ * it without reading the file a second time.
4973
+ *
4974
+ * @return {any} The metadata.
4975
+ */
4976
+ describeParsed() {
4977
+ assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
4978
+ const header = this._header["QvdTableHeader"];
4979
+ const columns = this._allFields.map((field) => field["FieldName"]);
4980
+ const rowCount = headerInteger(header["NoOfRecords"]);
4981
+ validateRecordCount(rowCount, this._path, "readMetadata");
4982
+ const shape = new exports.QvdDataFrame([], columns, header, {
4983
+ symbolTableBytes: headerInteger(header["Offset"]),
4984
+ totalRows: rowCount,
4985
+ rowsLoaded: 0,
4986
+ symbolFiltering: false,
4987
+ symbolsKept: null
4988
+ });
4989
+ return {
4990
+ columns,
4991
+ rowCount,
4992
+ columnCount: columns.length,
4993
+ fields: columns.map((name) => shape.getFieldMetadata(name)),
4994
+ fileMetadata: shape.fileMetadata,
4995
+ metadata: header
4996
+ };
4997
+ }
4726
4998
  /**
4727
4999
  * What a read of this file would cost, and whether it fits, without reading it.
4728
5000
  *
@@ -4740,77 +5012,122 @@ var init_QvdFileReader = __esm({
4740
5012
  async checkRead(rawWindow, { chunkSize = null } = {}) {
4741
5013
  const window = normaliseWindow(rawWindow, this._path);
4742
5014
  return await this._closingAfter(async () => {
4743
- await this._readData({ offset: 0, limit: null }, true);
4744
- this._emitProgress("header", 0, 1);
4745
- await this._parseHeader();
4746
- this._emitProgress("header", 1, 1);
4747
- this._throwIfAborted();
4748
- assert4__default.default(
4749
- this._header && this._selectedFields && this._allFields && this._symbolTableOffset !== null,
4750
- "The QVD file header has not been parsed."
5015
+ await this._parseHeaderChecked();
5016
+ return this.checkParsed(window, { chunkSize });
5017
+ });
5018
+ }
5019
+ /**
5020
+ * Reads this file's header, and nothing else, leaving it parsed on the reader.
5021
+ *
5022
+ * What `checkRead` and `QvdFile` both start with: the second asks many questions of one header, so the
5023
+ * read that produces it is separate from the questions. Every check a read makes before it trusts the
5024
+ * header's numbers is made here, so that nothing downstream has to wonder whether they hold.
5025
+ *
5026
+ * @return {Promise<void>} When the header is parsed and checked.
5027
+ */
5028
+ async parseHeaderOnly() {
5029
+ return await this._closingAfter(async () => await this._parseHeaderChecked());
5030
+ }
5031
+ /**
5032
+ * `parseHeaderOnly`'s body, for a caller already inside a read session - `checkRead` is one.
5033
+ *
5034
+ * @return {Promise<void>} When the header is parsed and checked.
5035
+ * @private
5036
+ */
5037
+ async _parseHeaderChecked() {
5038
+ await this._readData({ offset: 0, limit: null }, true);
5039
+ this._emitProgress("header", 0, 1);
5040
+ await this._parseHeader();
5041
+ this._emitProgress("header", 1, 1);
5042
+ this._throwIfAborted();
5043
+ assert4__default.default(
5044
+ this._header && this._selectedFields && this._allFields && this._symbolTableOffset !== null,
5045
+ "The QVD file header has not been parsed."
5046
+ );
5047
+ const header = this._header["QvdTableHeader"];
5048
+ const totalRows = headerInteger(header["NoOfRecords"]);
5049
+ const recordSize = headerInteger(header["RecordByteSize"]);
5050
+ const symbolTableLength = headerInteger(header["Offset"]);
5051
+ validateRecordSize(recordSize, this._path, "checkRead");
5052
+ validateRecordCount(totalRows, this._path, "checkRead");
5053
+ if (!this._headerMatchesFile) {
5054
+ throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
5055
+ file: this._path,
5056
+ fileSize: this._fileSize,
5057
+ requiredBytes: this._symbolTableOffset + symbolTableLength + totalRows * recordSize,
5058
+ stage: "checkRead"
5059
+ });
5060
+ }
5061
+ const tableLength = this._symbolTableLength();
5062
+ for (const field of this._allFields) {
5063
+ validateFieldMetadata(field, tableLength, this._path);
5064
+ validateFieldBitMetadata(field, recordSize, this._path);
5065
+ }
5066
+ validateSymbolAreas(this._allFields, this._path);
5067
+ }
5068
+ /**
5069
+ * What a read of the parsed header's file would cost, with no I/O at all.
5070
+ *
5071
+ * Separate from `checkRead` because a `QvdFile` asks this of one header many times - once per page a
5072
+ * viewer scrolls to - and the header is already in hand. `parseHeaderOnly` has to have run.
5073
+ *
5074
+ * @param {QvdRowWindow} window The rows the read would cover, normalised.
5075
+ * @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean}} [options]
5076
+ * `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask about a read other than the
5077
+ * one this reader was built for, which is what a `QvdFile` does per call.
5078
+ * @return {any} The answer - see `checkMemory`.
5079
+ */
5080
+ checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0 } = {}) {
5081
+ assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
5082
+ const header = this._header["QvdTableHeader"];
5083
+ const totalRows = headerInteger(header["NoOfRecords"]);
5084
+ const recordSize = headerInteger(header["RecordByteSize"]);
5085
+ const symbolTableLength = headerInteger(header["Offset"]);
5086
+ const builds = materialisesRows === void 0 ? this._materialisesRows : materialisesRows;
5087
+ const selected = selectFields(this._allFields, fields === void 0 ? this._requestedFields : fields, this._path);
5088
+ const resolved = resolveWindow(window, totalRows);
5089
+ const windowRows = resolved.limit;
5090
+ const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
5091
+ const analysisAhead = this._analysisAhead(window, resolved, totalRows, symbolTableLength);
5092
+ const measured = getMemoryBudget();
5093
+ const ask = /* @__PURE__ */ __name((asked, rows) => {
5094
+ const bytes = symbolBytesOf(this._fieldsHeldAfter(asked, this._allFields), symbolTableLength);
5095
+ const read = symbolBytesOf(this._fieldsReadBy(asked), symbolTableLength);
5096
+ return checkMemory({
5097
+ measured,
5098
+ symbolTableSize: bytes,
5099
+ maxRows: rows,
5100
+ totalRows,
5101
+ safetyFactor: this._memorySafetyFactor,
5102
+ columnCount: asked.length,
5103
+ materialisesRows: builds,
5104
+ live: liveRows,
5105
+ // A paging read keeps whole columns, so it is charged for whole columns - see `estimateMemoryUsage`.
5106
+ wholeSymbols: this._symbolCache !== null,
5107
+ bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead),
5108
+ // What it reads from the file, which is not what it holds: the symbol areas it has still to read,
5109
+ // and every record the window covers, read a slice at a time and not kept - twice over where the
5110
+ // symbol-usage pass will run, since it reads them before the decode reads them again.
5111
+ readBytes: read + readPasses(analysisAhead) * windowRows * recordSize
5112
+ });
5113
+ }, "ask");
5114
+ const answer = ask(selected, windowRows);
5115
+ if (!answer.fits && selected.length > 1) {
5116
+ const bySize = [...selected].sort(
5117
+ (a, b) => headerInteger(a["Length"]) - headerInteger(b["Length"])
4751
5118
  );
4752
- const header = this._header["QvdTableHeader"];
4753
- const totalRows = headerInteger(header["NoOfRecords"]);
4754
- const recordSize = headerInteger(header["RecordByteSize"]);
4755
- const symbolTableLength = headerInteger(header["Offset"]);
4756
- const selected = this._selectedFields;
4757
- validateRecordSize(recordSize, this._path, "checkRead");
4758
- validateRecordCount(totalRows, this._path, "checkRead");
4759
- if (!this._headerMatchesFile) {
4760
- throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
4761
- file: this._path,
4762
- fileSize: this._fileSize,
4763
- requiredBytes: this._symbolTableOffset + symbolTableLength + totalRows * recordSize,
4764
- stage: "checkRead"
4765
- });
4766
- }
4767
- const tableLength = this._symbolTableLength();
4768
- for (const field of this._allFields) {
4769
- validateFieldMetadata(field, tableLength, this._path);
4770
- validateFieldBitMetadata(field, recordSize, this._path);
4771
- }
4772
- validateSymbolAreas(this._allFields, this._path);
4773
- const resolved = resolveWindow(window, totalRows);
4774
- const windowRows = resolved.limit;
4775
- const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
4776
- const analysisAhead = this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
4777
- const measured = getMemoryBudget();
4778
- const ask = /* @__PURE__ */ __name((fields, rows) => {
4779
- const bytes = symbolBytesOf(fields, symbolTableLength);
4780
- return checkMemory({
4781
- measured,
4782
- symbolTableSize: bytes,
4783
- maxRows: rows,
4784
- totalRows,
4785
- safetyFactor: this._memorySafetyFactor,
4786
- columnCount: fields.length,
4787
- materialisesRows: this._materialisesRows,
4788
- live: liveRows,
4789
- bytesHeld: this._bytesHeld(bytes, rows, recordSize, liveRows, analysisAhead),
4790
- // What it reads from the file, which is not what it holds: the symbol areas, and every record
4791
- // the window covers, read a slice at a time and not kept - twice over where the symbol-usage
4792
- // pass will run, since it reads them before the decode reads them again.
4793
- readBytes: bytes + readPasses(analysisAhead) * windowRows * recordSize
4794
- });
4795
- }, "ask");
4796
- const answer = ask(selected, windowRows);
4797
- if (!answer.fits && selected.length > 1) {
4798
- const bySize = [...selected].sort(
4799
- (a, b) => headerInteger(a["Length"]) - headerInteger(b["Length"])
4800
- );
4801
- for (let take = selected.length - 1; take >= 1; take -= 1) {
4802
- const fewer = bySize.slice(0, take);
4803
- if (ask(fewer, windowRows).fits) {
4804
- answer.suggestions.push({
4805
- option: "fields",
4806
- value: fewer.map((field) => field["FieldName"])
4807
- });
4808
- break;
4809
- }
5119
+ for (let take = selected.length - 1; take >= 1; take -= 1) {
5120
+ const fewer = bySize.slice(0, take);
5121
+ if (ask(fewer, windowRows).fits) {
5122
+ answer.suggestions.push({
5123
+ option: "fields",
5124
+ value: fewer.map((field) => field["FieldName"])
5125
+ });
5126
+ break;
4810
5127
  }
4811
5128
  }
4812
- return answer;
4813
- });
5129
+ }
5130
+ return answer;
4814
5131
  }
4815
5132
  /**
4816
5133
  * Loads the QVD file into memory and parses it.
@@ -4976,7 +5293,7 @@ var init_QvdFileReader = __esm({
4976
5293
  const rowsAvailable = resolved.limit;
4977
5294
  let symbolsToKeep = null;
4978
5295
  let symbolsKept = null;
4979
- if (this._analysisWouldRun(window, resolved, totalRows, symbolTableLength)) {
5296
+ if (this._analysisAhead(window, resolved, totalRows, symbolTableLength)) {
4980
5297
  symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
4981
5298
  symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
4982
5299
  }
@@ -5068,6 +5385,224 @@ var init_QvdFileReader = __esm({
5068
5385
  }
5069
5386
  });
5070
5387
 
5388
+ // src/QvdFile.js
5389
+ var QvdFile_exports = {};
5390
+ __export(QvdFile_exports, {
5391
+ QvdFile: () => exports.QvdFile
5392
+ });
5393
+ exports.QvdFile = void 0;
5394
+ var init_QvdFile = __esm({
5395
+ "src/QvdFile.js"() {
5396
+ init_QvdErrors();
5397
+ init_readOptions();
5398
+ exports.QvdFile = class {
5399
+ static {
5400
+ __name(this, "QvdFile");
5401
+ }
5402
+ /**
5403
+ * Not called directly - `QvdDataFrame.open()` is the way in, because a `QvdFile` is only ever a file
5404
+ * whose header has been read, and a constructor cannot wait for that.
5405
+ *
5406
+ * @param {any} reader The reader holding the parsed header.
5407
+ * @param {any} metadata What `readMetadata()` returns for this file.
5408
+ * @param {any} options The options the file was opened with.
5409
+ * @private
5410
+ */
5411
+ constructor(reader, metadata, options) {
5412
+ this._reader = reader;
5413
+ this._metadata = metadata;
5414
+ this._options = options;
5415
+ this._closed = false;
5416
+ this._tail = Promise.resolve();
5417
+ this._readers = { rows: null, columns: null };
5418
+ }
5419
+ /**
5420
+ * The file's header and schema, as `QvdDataFrame.readMetadata()` returns them.
5421
+ *
5422
+ * Read when the file was opened, so this costs nothing and cannot fail.
5423
+ *
5424
+ * @return {any} The metadata.
5425
+ */
5426
+ get metadata() {
5427
+ return this._metadata;
5428
+ }
5429
+ /**
5430
+ * Whether `close()` has been called.
5431
+ *
5432
+ * @return {boolean} True once it has.
5433
+ */
5434
+ get closed() {
5435
+ return this._closed;
5436
+ }
5437
+ /**
5438
+ * What a read of this file would cost, and whether it fits - with no I/O at all.
5439
+ *
5440
+ * The same answer `QvdDataFrame.checkRead()` gives, from the header this file already holds, so a
5441
+ * viewer can size a page before asking for it without touching the disk.
5442
+ *
5443
+ * @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
5444
+ * as?: 'rows'|'columns', chunkSize?: number|null}} [options] The read being asked about - the same
5445
+ * bag `rows()` takes, plus `as` and `chunkSize` to say which shape of read it is.
5446
+ * @return {any} The answer - `fits`, `reason`, `estimate`, `budget`, `exact`, `suggestions`.
5447
+ * @throws {QvdValidationError} If the file is closed, or an option's value is not valid.
5448
+ */
5449
+ check(options = {}) {
5450
+ this._refuseWhenClosed("check");
5451
+ const { as = "rows", chunkSize = null } = options;
5452
+ if (as !== "rows" && as !== "columns") {
5453
+ throw new exports.QvdValidationError("as must be 'rows' or 'columns'", {
5454
+ provided: as,
5455
+ reason: "option",
5456
+ option: "as",
5457
+ value: as,
5458
+ file: this._options.path
5459
+ });
5460
+ }
5461
+ if (chunkSize !== null) {
5462
+ requireChunkSize(chunkSize, this._options.path);
5463
+ }
5464
+ const reader = this._readers[as] ?? this._reader;
5465
+ return reader.checkParsed(normaliseWindow(windowFrom(options), this._options.path), {
5466
+ chunkSize,
5467
+ // Resolved here rather than left to the reader, exactly as `_page` resolves it. A warm reader is
5468
+ // still holding the last page's selection, and `checkParsed` falls back to it - so a `check()`
5469
+ // naming no fields answered for whatever the previous page happened to name. On a four-column
5470
+ // file after a page naming one of them, it reported 27,490 bytes for a read that costs 108,160:
5471
+ // understating, which is the direction that approves a read the read then refuses.
5472
+ fields: options.fields === void 0 ? this._options.fields ?? null : options.fields,
5473
+ materialisesRows: as === "rows"
5474
+ });
5475
+ }
5476
+ /**
5477
+ * Reads a page of rows.
5478
+ *
5479
+ * @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
5480
+ * onProgress?: Function, signal?: AbortSignal}} [options] The page, and how to read it. One bag,
5481
+ * as every other entry point takes: `offset` and `limit` say which rows, `fields` names a projection
5482
+ * for this page alone, and anything left out falls back to what the file was opened with.
5483
+ * @return {Promise<any>} The page, as a `QvdDataFrame`.
5484
+ * @throws {QvdValidationError} If the file is closed.
5485
+ */
5486
+ async rows(options = {}) {
5487
+ this._refuseWhenClosed("rows");
5488
+ return await this._serialised(async () => await this._page(options, true, (reader, window) => reader.load(window)));
5489
+ }
5490
+ /**
5491
+ * Reads a page as columns, building no rows.
5492
+ *
5493
+ * @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
5494
+ * onProgress?: Function, signal?: AbortSignal}} [options] The page, as `rows()` takes it.
5495
+ * @return {Promise<any>} The page, as a `QvdColumnTable`.
5496
+ * @throws {QvdValidationError} If the file is closed.
5497
+ */
5498
+ async columns(options = {}) {
5499
+ this._refuseWhenClosed("columns");
5500
+ return await this._serialised(
5501
+ async () => await this._page(options, false, (reader, window) => reader.loadColumnar(window))
5502
+ );
5503
+ }
5504
+ /**
5505
+ * Closes the file.
5506
+ *
5507
+ * Every call after it is refused with `reason: 'closed'`. Calling it twice is not an error: a
5508
+ * `finally` that closes and an `await using` that closes are both right, and both may run.
5509
+ *
5510
+ * @return {Promise<void>} When the pages already in flight have finished.
5511
+ */
5512
+ async close() {
5513
+ if (this._closed) {
5514
+ return;
5515
+ }
5516
+ this._closed = true;
5517
+ await this._tail;
5518
+ for (const reader of Object.values(this._readers)) {
5519
+ reader?.endPaging();
5520
+ }
5521
+ this._readers = { rows: null, columns: null };
5522
+ this._reader.endPaging();
5523
+ this._reader = null;
5524
+ }
5525
+ /**
5526
+ * `await using` support, where the runtime has it.
5527
+ *
5528
+ * @return {Promise<void>} When closed.
5529
+ */
5530
+ async [Symbol.asyncDispose]() {
5531
+ await this.close();
5532
+ }
5533
+ /**
5534
+ * Refuses a call on a closed file, in the vocabulary the rest of the API uses.
5535
+ *
5536
+ * @param {string} call The method the caller reached for, for the error.
5537
+ * @private
5538
+ */
5539
+ _refuseWhenClosed(call) {
5540
+ if (this._closed) {
5541
+ throw new exports.QvdValidationError("The file is closed: open it again to read from it", {
5542
+ reason: "closed",
5543
+ call,
5544
+ file: this._options.path
5545
+ });
5546
+ }
5547
+ }
5548
+ /**
5549
+ * Runs `work` after everything asked for before it, and before everything asked for after.
5550
+ *
5551
+ * @param {() => Promise<any>} work The page to read.
5552
+ * @return {Promise<any>} Its result.
5553
+ * @private
5554
+ */
5555
+ async _serialised(work) {
5556
+ const run = this._tail.then(work, work);
5557
+ this._tail = run.then(
5558
+ () => void 0,
5559
+ () => void 0
5560
+ );
5561
+ return await run;
5562
+ }
5563
+ /**
5564
+ * Reads one page, through the reader that keeps what the pages before it decoded.
5565
+ *
5566
+ * One reader for every page rather than one per page, which is what makes the symbol cache possible:
5567
+ * decoding the symbols is 72% of a page of a hundred rows from a 300,000-row file, and a reader built
5568
+ * fresh each time did all of it again. Two readers, because a columnar page builds no rows and a row
5569
+ * page does, and `materialisesRows` is fixed when a reader is constructed - so each shape keeps its
5570
+ * own, and its own cache.
5571
+ *
5572
+ * Options that belong to one call rather than to the file - the fields this page alone wants, and the
5573
+ * `onProgress` and `signal` watching it - are told to that shared reader for the next read and no
5574
+ * further. A page naming one of them used to build a reader of its own instead, which quietly turned
5575
+ * the cache off and the two-pass symbol path on: watching a page changed what the page did.
5576
+ *
5577
+ * @param {any} options What the call passed.
5578
+ * @param {boolean} builds Whether the page materialises rows.
5579
+ * @param {(reader: any, window: any) => Promise<any>} read The read to make.
5580
+ * @return {Promise<any>} The page.
5581
+ * @private
5582
+ */
5583
+ async _page(options, builds, read) {
5584
+ const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
5585
+ const kept = builds ? "rows" : "columns";
5586
+ if (!this._readers[kept]) {
5587
+ const reader2 = new QvdFileReader2(this._options.path, {
5588
+ ...readerOptionsFrom(this._options),
5589
+ materialisesRows: builds
5590
+ });
5591
+ reader2.beginPaging();
5592
+ this._readers[kept] = reader2;
5593
+ }
5594
+ const reader = this._readers[kept];
5595
+ reader.selectForNextRead(options.fields === void 0 ? this._options.fields ?? null : options.fields);
5596
+ reader.observeNextRead({
5597
+ onProgress: options.onProgress ?? this._options.onProgress,
5598
+ signal: options.signal ?? this._options.signal
5599
+ });
5600
+ return await read(reader, windowFrom(options));
5601
+ }
5602
+ };
5603
+ }
5604
+ });
5605
+
5071
5606
  // src/QvdDataFrame.js
5072
5607
  function defaultFieldHeader(fieldName) {
5073
5608
  return {
@@ -5878,6 +6413,49 @@ var init_QvdDataFrame = __esm({
5878
6413
  const reader = new QvdFileReader2(path5, { ...readerOptionsFrom(options), materialisesRows: as === "rows" });
5879
6414
  return await reader.checkRead(windowFrom(options), { chunkSize });
5880
6415
  }
6416
+ /**
6417
+ * Opens a QVD file for paging, reading its header and nothing else.
6418
+ *
6419
+ * Every other entry point is one read from start to finish. A viewer showing a hundred rows at a time
6420
+ * pays the header again on every page, and `iterate()` goes forwards only - it cannot jump to row five
6421
+ * million and it cannot go back. This holds the header so that `check()` costs nothing and a page can
6422
+ * be asked for by position.
6423
+ *
6424
+ * ```js
6425
+ * const qvd = await QvdDataFrame.open('sales.qvd', {allowedDir: '/data'});
6426
+ *
6427
+ * qvd.metadata; // read once, when it opened
6428
+ * const answer = qvd.check({offset: 0, limit: 100}); // no I/O at all
6429
+ * const page = await qvd.rows({offset: 5_000_000, limit: 100});
6430
+ * const cols = await qvd.columns({offset: 0, limit: 100, fields: ['Amount']});
6431
+ *
6432
+ * await qvd.close();
6433
+ * ```
6434
+ *
6435
+ * The header is read once, and so is each column: a column decoded for one page is kept for the pages
6436
+ * after it, so the first page costs about what a single read costs and the ones after it are cheap.
6437
+ * What a file has decoded is charged to the memory check, so a page is refused rather than the process
6438
+ * aborting, and `close()` releases it - close a file you have finished with.
6439
+ *
6440
+ * Each page still opens the file, and a first touch still decodes a whole column rather than only as
6441
+ * far as the page needs.
6442
+ *
6443
+ * @param {string} path The QVD file.
6444
+ * @param {object} [options] What `fromQvd()` takes - `allowedDir`, `fields`, `duals`,
6445
+ * `coerceNumericStrings`, `memorySafetyFactor` - describing the file and how its values read. A
6446
+ * window means nothing here: pages carry their own.
6447
+ * @return {Promise<import('./QvdFile.js').QvdFile>} The open file.
6448
+ * @throws {QvdValidationError} If an option's value is not valid, with `context.reason` of `option`.
6449
+ * @throws {QvdCorruptedError} If the header cannot be read, or describes a file this is not.
6450
+ */
6451
+ static async open(path5, options = {}) {
6452
+ const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
6453
+ const { QvdFile: QvdFile2 } = await Promise.resolve().then(() => (init_QvdFile(), QvdFile_exports));
6454
+ const reader = new QvdFileReader2(path5, readerOptionsFrom(options));
6455
+ reader.beginPaging();
6456
+ await reader.parseHeaderOnly();
6457
+ return new QvdFile2(reader, reader.describeParsed(), { ...options, path: path5 });
6458
+ }
5881
6459
  /**
5882
6460
  * Constructs a data frame from a dictionary.
5883
6461
  *
@@ -6182,6 +6760,7 @@ __name(dateToQlikSerial, "dateToQlikSerial");
6182
6760
  // src/index.js
6183
6761
  init_QvdDataFrame();
6184
6762
  init_QvdColumnTable();
6763
+ init_QvdFile();
6185
6764
  init_QvdFileReader();
6186
6765
  init_QvdFileWriter();
6187
6766
  init_QvdErrors();