qvdjs 2.0.6 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/index.cjs +1028 -133
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1029 -134
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -453,7 +453,9 @@ var init_optionTypes = __esm({
|
|
|
453
453
|
function requireRowCount(value, name, filePath) {
|
|
454
454
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
455
455
|
throw new exports.QvdValidationError(`${name} must be a non-negative integer`, {
|
|
456
|
+
reason: "option",
|
|
456
457
|
option: name,
|
|
458
|
+
value,
|
|
457
459
|
provided: value,
|
|
458
460
|
type: typeof value,
|
|
459
461
|
file: filePath
|
|
@@ -470,6 +472,9 @@ function normaliseWindow(window, filePath) {
|
|
|
470
472
|
}
|
|
471
473
|
if (typeof window !== "object" || Array.isArray(window)) {
|
|
472
474
|
throw new exports.QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
|
|
475
|
+
reason: "option",
|
|
476
|
+
option: "limit",
|
|
477
|
+
value: window,
|
|
473
478
|
provided: window,
|
|
474
479
|
type: typeof window,
|
|
475
480
|
file: filePath
|
|
@@ -480,6 +485,9 @@ function normaliseWindow(window, filePath) {
|
|
|
480
485
|
const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
|
|
481
486
|
if (limitGiven && maxRowsGiven) {
|
|
482
487
|
throw new exports.QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
|
|
488
|
+
reason: "option",
|
|
489
|
+
option: "limit",
|
|
490
|
+
value: limit,
|
|
483
491
|
maxRows,
|
|
484
492
|
limit,
|
|
485
493
|
file: filePath
|
|
@@ -490,6 +498,19 @@ function normaliseWindow(window, filePath) {
|
|
|
490
498
|
limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
|
|
491
499
|
};
|
|
492
500
|
}
|
|
501
|
+
function requireChunkSize(chunkSize, filePath) {
|
|
502
|
+
if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
503
|
+
throw new exports.QvdValidationError("chunkSize must be a positive integer", {
|
|
504
|
+
reason: "option",
|
|
505
|
+
option: "chunkSize",
|
|
506
|
+
value: chunkSize,
|
|
507
|
+
provided: chunkSize,
|
|
508
|
+
type: typeof chunkSize,
|
|
509
|
+
file: filePath
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
return chunkSize;
|
|
513
|
+
}
|
|
493
514
|
function resolveWindow(window, totalRows) {
|
|
494
515
|
const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
|
|
495
516
|
const offset = Math.min(window.offset, rows);
|
|
@@ -504,6 +525,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
504
525
|
}
|
|
505
526
|
if (!Array.isArray(requested)) {
|
|
506
527
|
throw new exports.QvdValidationError("fields must be an array of field names", {
|
|
528
|
+
reason: "option",
|
|
529
|
+
option: "fields",
|
|
530
|
+
value: requested,
|
|
507
531
|
provided: requested,
|
|
508
532
|
type: typeof requested,
|
|
509
533
|
file: filePath
|
|
@@ -512,6 +536,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
512
536
|
const available = fields.map((field) => field["FieldName"]);
|
|
513
537
|
if (requested.length === 0) {
|
|
514
538
|
throw new exports.QvdValidationError("fields must name at least one field", {
|
|
539
|
+
reason: "option",
|
|
540
|
+
option: "fields",
|
|
541
|
+
value: requested,
|
|
515
542
|
availableColumns: available,
|
|
516
543
|
file: filePath
|
|
517
544
|
});
|
|
@@ -520,6 +547,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
520
547
|
return requested.map((name) => {
|
|
521
548
|
if (typeof name !== "string") {
|
|
522
549
|
throw new exports.QvdValidationError("Field names must be strings", {
|
|
550
|
+
reason: "option",
|
|
551
|
+
option: "fields",
|
|
552
|
+
value: name,
|
|
523
553
|
provided: name,
|
|
524
554
|
type: typeof name,
|
|
525
555
|
availableColumns: available,
|
|
@@ -528,6 +558,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
528
558
|
}
|
|
529
559
|
if (seen.has(name)) {
|
|
530
560
|
throw new exports.QvdValidationError(`Field '${name}' is listed twice`, {
|
|
561
|
+
reason: "option",
|
|
562
|
+
option: "fields",
|
|
563
|
+
value: name,
|
|
531
564
|
column: name,
|
|
532
565
|
fields: requested,
|
|
533
566
|
file: filePath
|
|
@@ -537,6 +570,9 @@ function selectFields(fields, requested, filePath) {
|
|
|
537
570
|
const index = available.indexOf(name);
|
|
538
571
|
if (index === -1) {
|
|
539
572
|
throw new exports.QvdValidationError(`Column '${name}' does not exist`, {
|
|
573
|
+
reason: "option",
|
|
574
|
+
option: "fields",
|
|
575
|
+
value: name,
|
|
540
576
|
column: name,
|
|
541
577
|
availableColumns: available,
|
|
542
578
|
file: filePath
|
|
@@ -551,7 +587,9 @@ function normaliseDuals(value, filePath) {
|
|
|
551
587
|
}
|
|
552
588
|
if (!DUAL_MODES.includes(value)) {
|
|
553
589
|
throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
590
|
+
reason: "option",
|
|
554
591
|
option: "duals",
|
|
592
|
+
value,
|
|
555
593
|
provided: value,
|
|
556
594
|
file: filePath
|
|
557
595
|
});
|
|
@@ -590,6 +628,7 @@ var init_readOptions = __esm({
|
|
|
590
628
|
init_optionTypes();
|
|
591
629
|
__name(requireRowCount, "requireRowCount");
|
|
592
630
|
__name(normaliseWindow, "normaliseWindow");
|
|
631
|
+
__name(requireChunkSize, "requireChunkSize");
|
|
593
632
|
__name(resolveWindow, "resolveWindow");
|
|
594
633
|
__name(selectFields, "selectFields");
|
|
595
634
|
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
@@ -2158,13 +2197,13 @@ function estimateRowMemory(rows, columnCount) {
|
|
|
2158
2197
|
}
|
|
2159
2198
|
return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
|
|
2160
2199
|
}
|
|
2161
|
-
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
|
|
2200
|
+
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null, wholeSymbols = false) {
|
|
2162
2201
|
const FULL_PARSE_OVERHEAD = 6;
|
|
2163
2202
|
const MINIMAL_OVERHEAD = 0.01;
|
|
2164
2203
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
2165
2204
|
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
2166
2205
|
const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
|
|
2167
|
-
if (maxRows === null || maxRows >= totalRows) {
|
|
2206
|
+
if (maxRows === null || maxRows >= totalRows || wholeSymbols) {
|
|
2168
2207
|
return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
|
|
2169
2208
|
}
|
|
2170
2209
|
const rowPercentage = maxRows / totalRows;
|
|
@@ -2173,8 +2212,8 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
2173
2212
|
const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
|
|
2174
2213
|
return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
|
|
2175
2214
|
}
|
|
2176
|
-
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
|
|
2177
|
-
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
2215
|
+
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false, wholeSymbols = false) {
|
|
2216
|
+
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows, null, wholeSymbols) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
2178
2217
|
if (costOf(totalRows) <= budget) {
|
|
2179
2218
|
return totalRows;
|
|
2180
2219
|
}
|
|
@@ -2193,11 +2232,19 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
|
|
|
2193
2232
|
}
|
|
2194
2233
|
return low;
|
|
2195
2234
|
}
|
|
2196
|
-
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
|
|
2235
|
+
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false, wholeSymbols = false) {
|
|
2197
2236
|
const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
|
|
2198
2237
|
const fits = /* @__PURE__ */ __name((chunk) => {
|
|
2199
2238
|
const live = Math.min(chunk * liveRowsPerChunk, covered);
|
|
2200
|
-
const cost = estimateMemoryUsage(
|
|
2239
|
+
const cost = estimateMemoryUsage(
|
|
2240
|
+
symbolTableSize,
|
|
2241
|
+
windowRows,
|
|
2242
|
+
totalRows,
|
|
2243
|
+
columnCount,
|
|
2244
|
+
true,
|
|
2245
|
+
chunk * liveRowsPerChunk,
|
|
2246
|
+
wholeSymbols
|
|
2247
|
+
) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
|
|
2201
2248
|
return cost <= budget;
|
|
2202
2249
|
}, "fits");
|
|
2203
2250
|
if (fits(covered)) {
|
|
@@ -2218,22 +2265,66 @@ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, col
|
|
|
2218
2265
|
}
|
|
2219
2266
|
return low;
|
|
2220
2267
|
}
|
|
2221
|
-
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld =
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2268
|
+
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null, bytesHeld = null, readBytes = null, wholeSymbols = false) {
|
|
2269
|
+
const answer = checkMemory({
|
|
2270
|
+
wholeSymbols,
|
|
2271
|
+
symbolTableSize,
|
|
2272
|
+
maxRows,
|
|
2273
|
+
totalRows,
|
|
2274
|
+
safetyFactor,
|
|
2275
|
+
columnCount,
|
|
2276
|
+
materialisesRows,
|
|
2277
|
+
live,
|
|
2278
|
+
bytesHeld,
|
|
2279
|
+
readBytes
|
|
2280
|
+
});
|
|
2281
|
+
if (answer.fits) {
|
|
2226
2282
|
return;
|
|
2227
2283
|
}
|
|
2228
|
-
const
|
|
2284
|
+
const { message, context } = answer.refusal;
|
|
2285
|
+
throw new exports.QvdValidationError(message, {
|
|
2286
|
+
file: filePath,
|
|
2287
|
+
...context,
|
|
2288
|
+
reason: "memory",
|
|
2289
|
+
check: answerOf(answer)
|
|
2290
|
+
});
|
|
2291
|
+
}
|
|
2292
|
+
function checkMemory({
|
|
2293
|
+
symbolTableSize,
|
|
2294
|
+
maxRows,
|
|
2295
|
+
totalRows,
|
|
2296
|
+
safetyFactor = 0.8,
|
|
2297
|
+
columnCount = 0,
|
|
2298
|
+
materialisesRows = true,
|
|
2299
|
+
live = null,
|
|
2300
|
+
bytesHeld = null,
|
|
2301
|
+
readBytes = null,
|
|
2302
|
+
measured = null,
|
|
2303
|
+
wholeSymbols = false
|
|
2304
|
+
}) {
|
|
2305
|
+
if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
|
|
2306
|
+
throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", {
|
|
2307
|
+
safetyFactor,
|
|
2308
|
+
reason: "option",
|
|
2309
|
+
option: "memorySafetyFactor",
|
|
2310
|
+
value: safetyFactor
|
|
2311
|
+
});
|
|
2312
|
+
}
|
|
2313
|
+
const budget = measured ?? getMemoryBudget();
|
|
2229
2314
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
2230
2315
|
const rowsLive = live === null ? null : live.rows;
|
|
2231
2316
|
const liveRowsPerChunk = live === null ? 1 : live.perChunk;
|
|
2232
2317
|
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
2233
|
-
const heapMemory = estimateMemoryUsage(
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2318
|
+
const heapMemory = estimateMemoryUsage(
|
|
2319
|
+
symbolTableSize,
|
|
2320
|
+
maxRows,
|
|
2321
|
+
totalRows,
|
|
2322
|
+
columnCount,
|
|
2323
|
+
materialisesRows,
|
|
2324
|
+
rowsLive,
|
|
2325
|
+
wholeSymbols
|
|
2326
|
+
);
|
|
2327
|
+
const { held, forRows: heldForRows, forChunk: heldForChunk } = bytesHeld ?? noBytesHeld;
|
|
2237
2328
|
const externalMemory = estimateExternalMemory(liveRows, columnCount) + held;
|
|
2238
2329
|
const bounded = budget.candidates.map((candidate) => {
|
|
2239
2330
|
const heapOnly = candidate.source === "V8 heap limit";
|
|
@@ -2245,11 +2336,26 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2245
2336
|
bounds: heapOnly ? "the V8 heap" : "the whole process"
|
|
2246
2337
|
};
|
|
2247
2338
|
});
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2339
|
+
if (safetyFactor === 0) {
|
|
2340
|
+
const lowest = budget.candidates.reduce((least, candidate) => candidate.bytes < least.bytes ? candidate : least);
|
|
2341
|
+
return {
|
|
2342
|
+
fits: true,
|
|
2343
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2344
|
+
// The ceilings are still there and still named - what is missing is any measurement against them.
|
|
2345
|
+
// An earlier version reported `bound: 'none'` here, a third value in a two-value vocabulary that a
|
|
2346
|
+
// caller switching on the documented two would fall straight through.
|
|
2347
|
+
budget: {
|
|
2348
|
+
...budgetOf(budget, { ...lowest, heapOnly: lowest.source === "V8 heap limit" }, 0),
|
|
2349
|
+
allowedBytes: Infinity
|
|
2350
|
+
},
|
|
2351
|
+
exact: symbolTableSize === 0,
|
|
2352
|
+
suggestions: []
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
const tightest = bounded.reduce(
|
|
2356
|
+
(worst, candidate) => candidate.needs / candidate.allowed > worst.needs / worst.allowed ? candidate : worst
|
|
2252
2357
|
);
|
|
2358
|
+
const binding = tightest.needs > tightest.allowed ? tightest : null;
|
|
2253
2359
|
const heapLimit = getHeapLimit();
|
|
2254
2360
|
const availableMemory = binding ? binding.bytes : budget.bytes;
|
|
2255
2361
|
const estimatedMemory = binding ? binding.needs : heapMemory;
|
|
@@ -2263,7 +2369,8 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2263
2369
|
totalRows,
|
|
2264
2370
|
columnCount,
|
|
2265
2371
|
materialisesRows,
|
|
2266
|
-
includeExternal
|
|
2372
|
+
includeExternal,
|
|
2373
|
+
wholeSymbols
|
|
2267
2374
|
), "fitting");
|
|
2268
2375
|
const firstGuess = fitting(liveRows);
|
|
2269
2376
|
const over = fitting(firstGuess);
|
|
@@ -2289,7 +2396,8 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2289
2396
|
totalRows,
|
|
2290
2397
|
columnCount,
|
|
2291
2398
|
liveRowsPerChunk,
|
|
2292
|
-
includeExternal
|
|
2399
|
+
includeExternal,
|
|
2400
|
+
wholeSymbols
|
|
2293
2401
|
), "chunkFitting");
|
|
2294
2402
|
const callersChunk = chunked ? Math.max(1, Math.floor(rowsLive / Math.max(1, liveRowsPerChunk))) : 0;
|
|
2295
2403
|
const firstChunk = chunked ? chunkFitting(callersChunk) : 0;
|
|
@@ -2306,42 +2414,94 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
2306
2414
|
} else {
|
|
2307
2415
|
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
2308
2416
|
}
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
{
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
heapLimitMB,
|
|
2318
|
-
reportedHeapLimitMB,
|
|
2319
|
-
availableRamMB,
|
|
2320
|
-
limitingFactor,
|
|
2321
|
-
limitingScope,
|
|
2322
|
-
memoryBudget: budget.candidates,
|
|
2323
|
-
memoryObserved: budget.observed,
|
|
2324
|
-
columnCount,
|
|
2325
|
-
totalRows,
|
|
2326
|
-
maxRows,
|
|
2327
|
-
recommendedMaxRows,
|
|
2328
|
-
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
2329
|
-
// recommendation for the other.
|
|
2330
|
-
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
2417
|
+
const suggestions = [];
|
|
2418
|
+
if (!nothingFits) {
|
|
2419
|
+
suggestions.push({ option: knob, value: recommendedValue });
|
|
2420
|
+
}
|
|
2421
|
+
if (!containerBound) {
|
|
2422
|
+
let needed = Math.ceil(estimatedMemory / safetyFactor / (1024 * 1024));
|
|
2423
|
+
while (Math.max(needed * 1024 * 1024, MINIMUM_BUDGET_BYTES) * safetyFactor < estimatedMemory) {
|
|
2424
|
+
needed += 1;
|
|
2331
2425
|
}
|
|
2332
|
-
|
|
2426
|
+
suggestions.push({ nodeOption: "--max-old-space-size", value: needed });
|
|
2427
|
+
}
|
|
2428
|
+
return {
|
|
2429
|
+
fits: false,
|
|
2430
|
+
reason: "memory",
|
|
2431
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2432
|
+
budget: budgetOf(budget, tightest, safetyFactor),
|
|
2433
|
+
// The symbol term is six times the bytes on disk, an overhead measured across files rather than
|
|
2434
|
+
// derived, so any read with symbols in it is an estimate and says so. Only a read that decodes
|
|
2435
|
+
// nothing can be exact.
|
|
2436
|
+
exact: symbolTableSize === 0,
|
|
2437
|
+
suggestions,
|
|
2438
|
+
refusal: {
|
|
2439
|
+
message: `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
2440
|
+
context: {
|
|
2441
|
+
symbolTableSize,
|
|
2442
|
+
symbolTableSizeMB: sizeMB,
|
|
2443
|
+
estimatedMemoryMB: estimatedMB,
|
|
2444
|
+
availableMemoryMB: availableMB,
|
|
2445
|
+
heapLimitMB,
|
|
2446
|
+
reportedHeapLimitMB,
|
|
2447
|
+
availableRamMB,
|
|
2448
|
+
limitingFactor,
|
|
2449
|
+
limitingScope,
|
|
2450
|
+
memoryBudget: budget.candidates,
|
|
2451
|
+
memoryObserved: budget.observed,
|
|
2452
|
+
columnCount,
|
|
2453
|
+
totalRows,
|
|
2454
|
+
maxRows,
|
|
2455
|
+
recommendedMaxRows,
|
|
2456
|
+
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
2457
|
+
// recommendation for the other.
|
|
2458
|
+
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
};
|
|
2333
2462
|
}
|
|
2463
|
+
return {
|
|
2464
|
+
fits: true,
|
|
2465
|
+
estimate: { heapBytes: heapMemory, externalBytes: externalMemory, readBytes },
|
|
2466
|
+
budget: budgetOf(budget, tightest, safetyFactor),
|
|
2467
|
+
exact: symbolTableSize === 0,
|
|
2468
|
+
suggestions: []
|
|
2469
|
+
};
|
|
2470
|
+
}
|
|
2471
|
+
function answerOf(answer) {
|
|
2472
|
+
const { refusal, ...rest } = answer;
|
|
2473
|
+
return rest;
|
|
2474
|
+
}
|
|
2475
|
+
function budgetOf(budget, tightest, safetyFactor) {
|
|
2476
|
+
const processLimit = budget.candidates.find((candidate) => candidate.source !== "V8 heap limit");
|
|
2477
|
+
return {
|
|
2478
|
+
heapBytes: usableOldSpaceLimit(),
|
|
2479
|
+
processBytes: processLimit ? processLimit.bytes : null,
|
|
2480
|
+
bound: tightest.heapOnly ? "heap" : "process",
|
|
2481
|
+
safetyFactor,
|
|
2482
|
+
allowedBytes: tightest.allowed ?? tightest.bytes * safetyFactor,
|
|
2483
|
+
candidates: budget.candidates,
|
|
2484
|
+
observed: budget.observed
|
|
2485
|
+
};
|
|
2334
2486
|
}
|
|
2335
2487
|
function formatCount(value) {
|
|
2336
2488
|
return value.toLocaleString("en-US");
|
|
2337
2489
|
}
|
|
2338
|
-
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
2490
|
+
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, wholeSymbols = false) {
|
|
2339
2491
|
const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
|
|
2340
2492
|
if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
2341
2493
|
return;
|
|
2342
2494
|
}
|
|
2343
2495
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
2344
|
-
const estimatedMemory = estimateMemoryUsage(
|
|
2496
|
+
const estimatedMemory = estimateMemoryUsage(
|
|
2497
|
+
symbolTableSize,
|
|
2498
|
+
maxRows,
|
|
2499
|
+
totalRows,
|
|
2500
|
+
columnCount,
|
|
2501
|
+
materialisesRows,
|
|
2502
|
+
null,
|
|
2503
|
+
wholeSymbols
|
|
2504
|
+
);
|
|
2345
2505
|
if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
2346
2506
|
return;
|
|
2347
2507
|
}
|
|
@@ -2352,7 +2512,7 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
2352
2512
|
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
2353
2513
|
);
|
|
2354
2514
|
}
|
|
2355
|
-
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
2515
|
+
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES, noBytesHeld;
|
|
2356
2516
|
var init_memoryUtils = __esm({
|
|
2357
2517
|
"src/util/memoryUtils.js"() {
|
|
2358
2518
|
init_QvdErrors();
|
|
@@ -2370,7 +2530,11 @@ var init_memoryUtils = __esm({
|
|
|
2370
2530
|
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
2371
2531
|
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
2372
2532
|
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
2533
|
+
noBytesHeld = Object.freeze({ held: 0, forRows: /* @__PURE__ */ __name(() => 0, "forRows"), forChunk: /* @__PURE__ */ __name(() => 0, "forChunk") });
|
|
2373
2534
|
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
2535
|
+
__name(checkMemory, "checkMemory");
|
|
2536
|
+
__name(answerOf, "answerOf");
|
|
2537
|
+
__name(budgetOf, "budgetOf");
|
|
2374
2538
|
__name(formatCount, "formatCount");
|
|
2375
2539
|
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
2376
2540
|
}
|
|
@@ -3455,6 +3619,36 @@ async function parseHeaderXml(text, file, stage) {
|
|
|
3455
3619
|
}
|
|
3456
3620
|
return parsed;
|
|
3457
3621
|
}
|
|
3622
|
+
function symbolBytesOf(selected, symbolTableLength) {
|
|
3623
|
+
const areaBytes = selected.map((field) => headerInteger(field["Length"]));
|
|
3624
|
+
return areaBytes.every((bytes) => Number.isSafeInteger(bytes) && bytes >= 0) ? Math.min(
|
|
3625
|
+
symbolTableLength,
|
|
3626
|
+
areaBytes.reduce((sum, bytes) => sum + bytes, 0)
|
|
3627
|
+
) : symbolTableLength;
|
|
3628
|
+
}
|
|
3629
|
+
function readPasses(analysisAhead) {
|
|
3630
|
+
return analysisAhead ? 2 : 1;
|
|
3631
|
+
}
|
|
3632
|
+
function validateWatchers(onProgress, signal, path5) {
|
|
3633
|
+
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
3634
|
+
throw new exports.QvdValidationError("onProgress must be a function", {
|
|
3635
|
+
provided: onProgress,
|
|
3636
|
+
type: typeof onProgress,
|
|
3637
|
+
reason: "option",
|
|
3638
|
+
option: "onProgress",
|
|
3639
|
+
file: path5
|
|
3640
|
+
});
|
|
3641
|
+
}
|
|
3642
|
+
if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
|
|
3643
|
+
throw new exports.QvdValidationError("signal must be an AbortSignal", {
|
|
3644
|
+
provided: signal,
|
|
3645
|
+
type: typeof signal,
|
|
3646
|
+
reason: "option",
|
|
3647
|
+
option: "signal",
|
|
3648
|
+
file: path5
|
|
3649
|
+
});
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3458
3652
|
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, SLICE_BYTES, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
|
|
3459
3653
|
var init_QvdFileReader = __esm({
|
|
3460
3654
|
"src/QvdFileReader.js"() {
|
|
@@ -3477,6 +3671,9 @@ var init_QvdFileReader = __esm({
|
|
|
3477
3671
|
COUNT_SYMBOLS_PAST = 65536;
|
|
3478
3672
|
__name(chunksFrom, "chunksFrom");
|
|
3479
3673
|
__name(parseHeaderXml, "parseHeaderXml");
|
|
3674
|
+
__name(symbolBytesOf, "symbolBytesOf");
|
|
3675
|
+
__name(readPasses, "readPasses");
|
|
3676
|
+
__name(validateWatchers, "validateWatchers");
|
|
3480
3677
|
exports.QvdFileReader = class {
|
|
3481
3678
|
static {
|
|
3482
3679
|
__name(this, "QvdFileReader");
|
|
@@ -3557,20 +3754,7 @@ var init_QvdFileReader = __esm({
|
|
|
3557
3754
|
});
|
|
3558
3755
|
}
|
|
3559
3756
|
this._sliceBytes = sliceBytes;
|
|
3560
|
-
|
|
3561
|
-
throw new exports.QvdValidationError("onProgress must be a function", {
|
|
3562
|
-
provided: onProgress,
|
|
3563
|
-
type: typeof onProgress,
|
|
3564
|
-
file: this._path
|
|
3565
|
-
});
|
|
3566
|
-
}
|
|
3567
|
-
if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
|
|
3568
|
-
throw new exports.QvdValidationError("signal must be an AbortSignal", {
|
|
3569
|
-
provided: signal,
|
|
3570
|
-
type: typeof signal,
|
|
3571
|
-
file: this._path
|
|
3572
|
-
});
|
|
3573
|
-
}
|
|
3757
|
+
validateWatchers(onProgress, signal, this._path);
|
|
3574
3758
|
this._requestedFields = fields === void 0 ? null : fields;
|
|
3575
3759
|
this._onProgress = onProgress;
|
|
3576
3760
|
this._signal = signal;
|
|
@@ -3579,6 +3763,8 @@ var init_QvdFileReader = __esm({
|
|
|
3579
3763
|
this._failed = null;
|
|
3580
3764
|
this._reading = false;
|
|
3581
3765
|
this._symbolAreas = null;
|
|
3766
|
+
this._symbolCache = null;
|
|
3767
|
+
this._cachedFor = null;
|
|
3582
3768
|
this._headerOffset = null;
|
|
3583
3769
|
this._symbolTableOffset = null;
|
|
3584
3770
|
this._indexTableOffset = null;
|
|
@@ -3590,6 +3776,7 @@ var init_QvdFileReader = __esm({
|
|
|
3590
3776
|
this._indexColumns = null;
|
|
3591
3777
|
this._rowsDecoded = 0;
|
|
3592
3778
|
this._fileSize = null;
|
|
3779
|
+
this._fileIdentity = null;
|
|
3593
3780
|
this._headerMatchesFile = false;
|
|
3594
3781
|
}
|
|
3595
3782
|
/**
|
|
@@ -3818,6 +4005,16 @@ var init_QvdFileReader = __esm({
|
|
|
3818
4005
|
const indexTableOffset = symbolTableOffset + symbolTableLength;
|
|
3819
4006
|
const recordSize = headerInteger(headerObj["QvdTableHeader"]["RecordByteSize"]);
|
|
3820
4007
|
const totalRows = headerInteger(headerObj["QvdTableHeader"]["NoOfRecords"]);
|
|
4008
|
+
const { size: fileSize, ino, dev, mtimeMs } = await handle.stat().catch(failed);
|
|
4009
|
+
this._fileSize = fileSize;
|
|
4010
|
+
this._fileIdentity = `${dev}:${ino}:${mtimeMs}:${fileSize}`;
|
|
4011
|
+
this._headerMatchesFile = false;
|
|
4012
|
+
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
4013
|
+
(value) => Number.isSafeInteger(value) && value >= 0
|
|
4014
|
+
);
|
|
4015
|
+
if (headerNumbersUsable) {
|
|
4016
|
+
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
|
|
4017
|
+
}
|
|
3821
4018
|
if (headerOnly) {
|
|
3822
4019
|
this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3823
4020
|
this._emitProgress("read", 1, 1);
|
|
@@ -3826,20 +4023,8 @@ var init_QvdFileReader = __esm({
|
|
|
3826
4023
|
this._headerBuffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3827
4024
|
const selected = selectFields(headerFields, this._requestedFields, this._path);
|
|
3828
4025
|
const columnCount = selected.length;
|
|
3829
|
-
const
|
|
3830
|
-
const
|
|
3831
|
-
symbolTableLength,
|
|
3832
|
-
areaBytes.reduce((sum, bytes) => sum + bytes, 0)
|
|
3833
|
-
) : symbolTableLength;
|
|
3834
|
-
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
3835
|
-
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3836
|
-
);
|
|
3837
|
-
const { size: fileSize } = await handle.stat().catch(failed);
|
|
3838
|
-
this._fileSize = fileSize;
|
|
3839
|
-
this._headerMatchesFile = false;
|
|
3840
|
-
if (headerNumbersUsable) {
|
|
3841
|
-
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
|
|
3842
|
-
}
|
|
4026
|
+
const symbolBytes = symbolBytesOf(this._fieldsHeldAfter(selected, headerFields), symbolTableLength);
|
|
4027
|
+
const readSymbolBytes = symbolBytesOf(this._fieldsReadBy(selected), symbolTableLength);
|
|
3843
4028
|
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3844
4029
|
const windowRows = resolved.limit;
|
|
3845
4030
|
if (headerNumbersUsable && this._headerMatchesFile) {
|
|
@@ -3853,12 +4038,23 @@ var init_QvdFileReader = __esm({
|
|
|
3853
4038
|
this._materialisesRows,
|
|
3854
4039
|
liveRows,
|
|
3855
4040
|
this._bytesHeld(
|
|
3856
|
-
|
|
4041
|
+
readSymbolBytes,
|
|
3857
4042
|
windowRows,
|
|
3858
4043
|
recordSize,
|
|
3859
4044
|
liveRows,
|
|
3860
|
-
this.
|
|
3861
|
-
)
|
|
4045
|
+
this._analysisAhead(window, resolved, totalRows, symbolTableLength)
|
|
4046
|
+
),
|
|
4047
|
+
// What it reads, which is not what it holds - the records go through one buffer and are not
|
|
4048
|
+
// kept. Carried so that a refusal's `check` says everything the pre-flight would have said.
|
|
4049
|
+
//
|
|
4050
|
+
// Twice over where the symbol-usage pass is still ahead of it: that pass reads the window's
|
|
4051
|
+
// records to find which symbols the rows use, and the decode then reads them again. Counted
|
|
4052
|
+
// once, the figure understated the I/O of exactly the reads that do the most of it.
|
|
4053
|
+
readSymbolBytes + readPasses(this._analysisAhead(window, resolved, totalRows, symbolTableLength)) * windowRows * recordSize,
|
|
4054
|
+
// A paging read keeps whole columns, so the estimate must not discount its symbols as a window's
|
|
4055
|
+
// sample of them - see `estimateMemoryUsage`. Under-charging is the direction that ends in a
|
|
4056
|
+
// heap-limit abort rather than an error.
|
|
4057
|
+
this._symbolCache !== null
|
|
3862
4058
|
);
|
|
3863
4059
|
}
|
|
3864
4060
|
if (window.offset === 0 && window.limit === null) {
|
|
@@ -3927,8 +4123,9 @@ var init_QvdFileReader = __esm({
|
|
|
3927
4123
|
*
|
|
3928
4124
|
* The areas are counted whole, although the read lets each range go once its fields are parsed, because
|
|
3929
4125
|
* two ranges are both live when a field of one is parsed between two fields of the other - the order the
|
|
3930
|
-
* caller asked for the fields decides it, so the sum is what holds in every order. The slice is
|
|
3931
|
-
* `_forEachSlice` will allocate
|
|
4126
|
+
* caller asked for the fields decides it, so the sum is what holds in every order. The slice is the
|
|
4127
|
+
* buffer `_forEachSlice` will allocate, sized by `_sliceRowsFor` so that the charge and the allocation
|
|
4128
|
+
* are one expression rather than two that agree today.
|
|
3932
4129
|
*
|
|
3933
4130
|
* @param {number} symbolBytes Bytes of symbols the read will read.
|
|
3934
4131
|
* @param {number} rows Records it will read.
|
|
@@ -3937,8 +4134,24 @@ var init_QvdFileReader = __esm({
|
|
|
3937
4134
|
* @private
|
|
3938
4135
|
*/
|
|
3939
4136
|
_bytesHeldBy(symbolBytes, rows, recordSize) {
|
|
3940
|
-
const
|
|
3941
|
-
return symbolBytes +
|
|
4137
|
+
const usable = Number.isSafeInteger(rows) && Number.isSafeInteger(recordSize) && rows > 0 && recordSize > 0;
|
|
4138
|
+
return symbolBytes + (usable ? this._sliceRowsFor(rows, recordSize) * recordSize : 0);
|
|
4139
|
+
}
|
|
4140
|
+
/**
|
|
4141
|
+
* Records the one buffer holds while a read of `rowCount` records goes through it.
|
|
4142
|
+
*
|
|
4143
|
+
* A slice is `sliceBytes` of records, rounded down to a whole record, or every record the read has left
|
|
4144
|
+
* when that is fewer - and at least one, since a read of a record wider than `sliceBytes` still has to
|
|
4145
|
+
* hold that record. The single definition: `_forEachSlice` allocates from it and the memory guard is
|
|
4146
|
+
* charged from it, so a change to how a read slices cannot leave the guard pricing the old rule.
|
|
4147
|
+
*
|
|
4148
|
+
* @param {number} rowCount Records the read will read.
|
|
4149
|
+
* @param {number} recordSize Bytes per record.
|
|
4150
|
+
* @return {number} Records in one slice.
|
|
4151
|
+
* @private
|
|
4152
|
+
*/
|
|
4153
|
+
_sliceRowsFor(rowCount, recordSize) {
|
|
4154
|
+
return Math.max(1, Math.min(rowCount, Math.floor(this._sliceBytes / Math.max(1, recordSize))));
|
|
3942
4155
|
}
|
|
3943
4156
|
/**
|
|
3944
4157
|
* What a read holds in bytes of the file, for the memory guard: what it holds now, and what a read
|
|
@@ -4026,6 +4239,191 @@ var init_QvdFileReader = __esm({
|
|
|
4026
4239
|
const chunkRows = liveRows === null ? windowRows : Math.max(1, Math.floor(liveRows.rows / Math.max(1, liveRows.perChunk)));
|
|
4027
4240
|
return analysisAhead ? Math.max(windowRows, chunkRows) : chunkRows;
|
|
4028
4241
|
}
|
|
4242
|
+
/**
|
|
4243
|
+
* The fields this reader will be holding the symbols of once this read has finished.
|
|
4244
|
+
*
|
|
4245
|
+
* The ones it selects, and - while paging - the ones it decoded for an earlier page and kept. That
|
|
4246
|
+
* union is what the memory checks have to be sized by, because it is what is live: a reader four
|
|
4247
|
+
* pages into a wide file holds four columns' values whether or not this page asks about them, and a
|
|
4248
|
+
* check sized by this page alone would approve a fifth column that does not fit beside them.
|
|
4249
|
+
*
|
|
4250
|
+
* The same set for every check, so the ceilings, the guard and the pre-flight cannot disagree about
|
|
4251
|
+
* what a paging read costs. Without a cache it is just the selection, which is what every one-shot
|
|
4252
|
+
* read has always been sized by.
|
|
4253
|
+
*
|
|
4254
|
+
* @param {Array<any>} selected The fields this read selects.
|
|
4255
|
+
* @param {Array<any>} all Every field in the header, to find a cached one by name.
|
|
4256
|
+
* @return {Array<any>} The fields whose symbols will be live.
|
|
4257
|
+
* @private
|
|
4258
|
+
*/
|
|
4259
|
+
_fieldsHeldAfter(selected, all) {
|
|
4260
|
+
if (this._symbolCache === null || this._symbolCache.size === 0) {
|
|
4261
|
+
return selected;
|
|
4262
|
+
}
|
|
4263
|
+
const names = new Set(selected.map((field) => field["FieldName"]));
|
|
4264
|
+
const cached = all.filter(
|
|
4265
|
+
(field) => !names.has(field["FieldName"]) && this._symbolCache !== null && this._symbolCache.has(field["FieldName"])
|
|
4266
|
+
);
|
|
4267
|
+
return [...selected, ...cached];
|
|
4268
|
+
}
|
|
4269
|
+
/**
|
|
4270
|
+
* The fields whose symbol areas this read will actually read.
|
|
4271
|
+
*
|
|
4272
|
+
* The selection, less anything already decoded and kept. `_fieldsHeldAfter` answers what the read will
|
|
4273
|
+
* be *holding*, which is the right figure for the heap; this is the right one for the bytes it buffers
|
|
4274
|
+
* while parsing and for the I/O it reports, because a cached column's area is left out of the plan
|
|
4275
|
+
* entirely and never read.
|
|
4276
|
+
*
|
|
4277
|
+
* Sized by the wrong one of the two, a warm page was charged external bytes for buffers it never
|
|
4278
|
+
* allocates - and external bytes bind against a container limit, so a page that fits could be refused -
|
|
4279
|
+
* and `estimate.readBytes` claimed I/O it does not perform: on four columns with three cached it
|
|
4280
|
+
* reported 3,155,600 bytes for a read of 788,930.
|
|
4281
|
+
*
|
|
4282
|
+
* @param {Array<any>} selected The fields this read selects.
|
|
4283
|
+
* @return {Array<any>} The fields whose areas will be read.
|
|
4284
|
+
* @private
|
|
4285
|
+
*/
|
|
4286
|
+
_fieldsReadBy(selected) {
|
|
4287
|
+
if (this._symbolCache === null || this._symbolCache.size === 0) {
|
|
4288
|
+
return selected;
|
|
4289
|
+
}
|
|
4290
|
+
return selected.filter(
|
|
4291
|
+
(field) => this._symbolCache !== null && !this._symbolCache.has(field["FieldName"])
|
|
4292
|
+
);
|
|
4293
|
+
}
|
|
4294
|
+
/**
|
|
4295
|
+
* Keeps what this reader decodes, so that a later read of the same file does not decode it again.
|
|
4296
|
+
*
|
|
4297
|
+
* For a caller reading one file many times over - a `QvdFile` and its pages - and off by default,
|
|
4298
|
+
* because every other entry point is one read and would only be holding values nobody will ask for
|
|
4299
|
+
* again. Decoding the symbols is 72% of a page of a hundred rows from a 300,000-row file; the rest is
|
|
4300
|
+
* the open, the header, the records and the rows.
|
|
4301
|
+
*
|
|
4302
|
+
* It turns the two-pass symbol path off with it. That path decodes only the symbols a window's rows
|
|
4303
|
+
* use, which is right for one read and wrong for a cache: a later page asking for a row that uses a
|
|
4304
|
+
* skipped symbol would read `undefined` where the value is. So a cached field is always a whole
|
|
4305
|
+
* field, walked and checked against its `NoOfSymbols` like any other.
|
|
4306
|
+
*
|
|
4307
|
+
* @return {void}
|
|
4308
|
+
*/
|
|
4309
|
+
beginPaging() {
|
|
4310
|
+
this._symbolCache = /* @__PURE__ */ new Map();
|
|
4311
|
+
}
|
|
4312
|
+
/**
|
|
4313
|
+
* Reads with the fields the caller names for this read alone, rather than the reader's own.
|
|
4314
|
+
*
|
|
4315
|
+
* A `QvdFile` is opened once and its pages may each name a projection, so the selection cannot be
|
|
4316
|
+
* fixed at construction as it is for every other entry point.
|
|
4317
|
+
*
|
|
4318
|
+
* It holds until the next call replaces it rather than being cleared by the read, so **every caller
|
|
4319
|
+
* sets it before every read**, passing the file's own fields where the page named none. A caller that
|
|
4320
|
+
* relied on it being empty would instead get the projection of whatever ran last: that is what made a
|
|
4321
|
+
* `check()` naming no fields answer for the previous page's columns.
|
|
4322
|
+
*
|
|
4323
|
+
* @param {Array<string>|null|undefined} fields The fields, or undefined to use the reader's own.
|
|
4324
|
+
* @return {void}
|
|
4325
|
+
*/
|
|
4326
|
+
selectForNextRead(fields) {
|
|
4327
|
+
if (fields !== void 0) {
|
|
4328
|
+
this._requestedFields = fields;
|
|
4329
|
+
}
|
|
4330
|
+
}
|
|
4331
|
+
/**
|
|
4332
|
+
* Watches the next read with the caller's `onProgress` and `signal`, rather than the reader's own.
|
|
4333
|
+
*
|
|
4334
|
+
* Both belong to one call, and a reader is told them when it is built - so a `QvdFile` page that named
|
|
4335
|
+
* either used to get a reader of its own. That made passing a progress callback change what the read
|
|
4336
|
+
* did rather than only observing it: a fresh reader is not paging, so it took the two-pass symbol
|
|
4337
|
+
* path, reported a different `loadStats.symbolFiltering`, and cached nothing. An observer must not
|
|
4338
|
+
* change what it observes, and a caller must not have to choose between cancelling a page and paging
|
|
4339
|
+
* cheaply.
|
|
4340
|
+
*
|
|
4341
|
+
* Like `selectForNextRead`, it holds until the next call replaces it rather than being cleared by the
|
|
4342
|
+
* read, so a caller that sets it for one page and not the next is still watched on the next - pass the
|
|
4343
|
+
* file's own watchers explicitly, as `QvdFile._page` does, rather than leaving them out.
|
|
4344
|
+
*
|
|
4345
|
+
* @param {{onProgress?: Function, signal?: AbortSignal}} [watchers] What this read is watched with.
|
|
4346
|
+
* @return {void}
|
|
4347
|
+
*/
|
|
4348
|
+
observeNextRead({ onProgress, signal } = {}) {
|
|
4349
|
+
validateWatchers(onProgress, signal, this._path);
|
|
4350
|
+
this._onProgress = onProgress;
|
|
4351
|
+
this._signal = signal;
|
|
4352
|
+
}
|
|
4353
|
+
/**
|
|
4354
|
+
* Whether the symbol-usage pass runs for this read, cache and all.
|
|
4355
|
+
*
|
|
4356
|
+
* `_analysisWouldRun` answers whether the window wants the pass; a paging read never takes it, because
|
|
4357
|
+
* a column decoded in part cannot be kept. Asked in one place because it was asked in two and they
|
|
4358
|
+
* disagreed: the pass was gated on the cache while the memory charge and `estimate.readBytes` were
|
|
4359
|
+
* not, so every page of a file above the threshold was charged a slice of records it never buffered
|
|
4360
|
+
* and reported twice the bytes it read.
|
|
4361
|
+
*
|
|
4362
|
+
* @param {QvdRowWindow} window The window as the caller spelled it.
|
|
4363
|
+
* @param {{offset: number, limit: number}} resolved Where it lands in this file.
|
|
4364
|
+
* @param {number} totalRows Rows the file declares.
|
|
4365
|
+
* @param {number} symbolTableLength The symbol table's declared length.
|
|
4366
|
+
* @return {boolean} Whether the pass will run.
|
|
4367
|
+
* @private
|
|
4368
|
+
*/
|
|
4369
|
+
_analysisAhead(window, resolved, totalRows, symbolTableLength) {
|
|
4370
|
+
return this._symbolCache === null && this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
|
|
4371
|
+
}
|
|
4372
|
+
/**
|
|
4373
|
+
* Empties the cache when the header in front of us is not the one it was decoded from.
|
|
4374
|
+
*
|
|
4375
|
+
* The fingerprint is what a rewrite moves: the file's identity on disk - device, inode, modification
|
|
4376
|
+
* time and size - and then the header numbers, down to each cached field's own offset, length and
|
|
4377
|
+
* symbol count.
|
|
4378
|
+
*
|
|
4379
|
+
* The header numbers alone were not enough, and the gap is not exotic. `QvdFileWriter` carries
|
|
4380
|
+
* `CreateUtcTime` over from the metadata it is handed, so reading a QVD, changing one text to another
|
|
4381
|
+
* of the same byte length and writing it back leaves `CreateUtcTime`, `NoOfRecords`, `Offset` and
|
|
4382
|
+
* every field's `Offset`, `Length` and `NoOfSymbols` exactly as they were - a different file the
|
|
4383
|
+
* fingerprint could not tell from the first. The filesystem sees it either way: an atomic write
|
|
4384
|
+
* renames a new file into place, which changes the inode, and an in-place one moves `mtimeMs`.
|
|
4385
|
+
*
|
|
4386
|
+
* @return {void}
|
|
4387
|
+
* @private
|
|
4388
|
+
*/
|
|
4389
|
+
_forgetCacheIfFileChanged() {
|
|
4390
|
+
if (this._symbolCache === null || this._symbolCache.size === 0) {
|
|
4391
|
+
return;
|
|
4392
|
+
}
|
|
4393
|
+
if (this._cachedFor !== this._fileFingerprint()) {
|
|
4394
|
+
this._symbolCache = /* @__PURE__ */ new Map();
|
|
4395
|
+
this._cachedFor = null;
|
|
4396
|
+
}
|
|
4397
|
+
}
|
|
4398
|
+
/**
|
|
4399
|
+
* What identifies the file this reader's cache was decoded from.
|
|
4400
|
+
*
|
|
4401
|
+
* @return {string} The fingerprint.
|
|
4402
|
+
* @private
|
|
4403
|
+
*/
|
|
4404
|
+
_fileFingerprint() {
|
|
4405
|
+
assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
4406
|
+
const header = this._header["QvdTableHeader"];
|
|
4407
|
+
return [
|
|
4408
|
+
// First, because it is the only part that moves when a rewrite preserves the header's numbers.
|
|
4409
|
+
this._fileIdentity,
|
|
4410
|
+
header["CreateUtcTime"],
|
|
4411
|
+
header["NoOfRecords"],
|
|
4412
|
+
header["Offset"],
|
|
4413
|
+
...this._allFields.map(
|
|
4414
|
+
(field) => `${field["FieldName"]}:${field["Offset"]}:${field["Length"]}:${field["NoOfSymbols"]}`
|
|
4415
|
+
)
|
|
4416
|
+
].join("|");
|
|
4417
|
+
}
|
|
4418
|
+
/**
|
|
4419
|
+
* Drops everything this reader has decoded, so that nothing outlives the caller that wanted it.
|
|
4420
|
+
*
|
|
4421
|
+
* @return {void}
|
|
4422
|
+
*/
|
|
4423
|
+
endPaging() {
|
|
4424
|
+
this._symbolCache = null;
|
|
4425
|
+
this._cachedFor = null;
|
|
4426
|
+
}
|
|
4029
4427
|
/**
|
|
4030
4428
|
* The symbol table's length, as much of it as the file holds: what the header declares, cut short where
|
|
4031
4429
|
* the file ends. Known before a byte of the table is read, so everything that can refuse the table is
|
|
@@ -4054,7 +4452,13 @@ var init_QvdFileReader = __esm({
|
|
|
4054
4452
|
* order, so ranges that touch are merged: a read of every field is one range, and so is a read of fields
|
|
4055
4453
|
* that happen to be neighbours. A read of one field of twenty reads that field's area alone.
|
|
4056
4454
|
*
|
|
4057
|
-
*
|
|
4455
|
+
* A field whose symbols this reader already holds is left out, because its bytes are not wanted: the
|
|
4456
|
+
* ranges are what gets read, and including a cached field's span had a page read every byte of every
|
|
4457
|
+
* column it named, cached or not. Measured on four columns of 20,000 distinct texts, a page naming all
|
|
4458
|
+
* four with three of them cached read all four columns' bytes - 1,155,600 of them, where 288,930 were
|
|
4459
|
+
* needed. The decode was saved and the I/O was not, which on the files #122 is about is the whole cost.
|
|
4460
|
+
*
|
|
4461
|
+
* Built once per read, from the fields the read must read, and each field's metadata is checked as it is
|
|
4058
4462
|
* added - a range is arithmetic on `Offset` and `Length`, and those have to be inside the table first.
|
|
4059
4463
|
* `_parseSymbolTable` checks every field of the file, selected or not, before it parses any.
|
|
4060
4464
|
*
|
|
@@ -4069,7 +4473,10 @@ var init_QvdFileReader = __esm({
|
|
|
4069
4473
|
}
|
|
4070
4474
|
assert4__default.default(this._selectedFields, "The QVD file fields have not been resolved before their symbols were read.");
|
|
4071
4475
|
const tableLength = this._symbolTableLength();
|
|
4072
|
-
const
|
|
4476
|
+
const toRead = this._selectedFields.filter(
|
|
4477
|
+
(field) => this._symbolCache === null || !this._symbolCache.has(field["FieldName"])
|
|
4478
|
+
);
|
|
4479
|
+
const areas = toRead.map((field) => {
|
|
4073
4480
|
validateFieldMetadata(field, tableLength, this._path);
|
|
4074
4481
|
const start = headerInteger(field["Offset"]);
|
|
4075
4482
|
return { field, start, end: start + headerInteger(field["Length"]) };
|
|
@@ -4154,7 +4561,7 @@ var init_QvdFileReader = __esm({
|
|
|
4154
4561
|
return;
|
|
4155
4562
|
}
|
|
4156
4563
|
assert4__default.default(this._indexTableOffset !== null, "The QVD file header has not been parsed before its records were read.");
|
|
4157
|
-
const sliceRows =
|
|
4564
|
+
const sliceRows = this._sliceRowsFor(rowCount, recordSize);
|
|
4158
4565
|
const slice = Buffer.alloc(sliceRows * recordSize);
|
|
4159
4566
|
const requiredBytes = this._indexTableOffset + (firstRow + rowCount) * recordSize;
|
|
4160
4567
|
for (let done = 0; done < rowCount; done += sliceRows) {
|
|
@@ -4391,9 +4798,11 @@ var init_QvdFileReader = __esm({
|
|
|
4391
4798
|
}
|
|
4392
4799
|
const allFields = this._allFields;
|
|
4393
4800
|
const fields = this._selectedFields;
|
|
4801
|
+
this._forgetCacheIfFileChanged();
|
|
4394
4802
|
const symbolTableSize = this._symbolTableLength();
|
|
4395
4803
|
const plan = this._symbolAreaPlan();
|
|
4396
|
-
const
|
|
4804
|
+
const readSymbolBytes = plan.ranges.reduce((sum, range) => sum + (range.end - range.start), 0);
|
|
4805
|
+
const symbolBytes = readSymbolBytes + symbolBytesOf(this._fieldsHeldAfter(fields, allFields).slice(fields.length), symbolTableSize);
|
|
4397
4806
|
const totalRows = headerInteger(this._header["QvdTableHeader"]["NoOfRecords"]);
|
|
4398
4807
|
const recordSize = headerInteger(this._header["QvdTableHeader"]["RecordByteSize"]);
|
|
4399
4808
|
validateSymbolTableSize(symbolBytes, this._path, totalRows);
|
|
@@ -4407,10 +4816,21 @@ var init_QvdFileReader = __esm({
|
|
|
4407
4816
|
fields.length,
|
|
4408
4817
|
this._materialisesRows,
|
|
4409
4818
|
liveRows,
|
|
4410
|
-
this._bytesHeld(
|
|
4819
|
+
this._bytesHeld(readSymbolBytes, rowsToLoad, recordSize, liveRows, false),
|
|
4820
|
+
// `symbolsToKeep` is non-null exactly when the symbol-usage pass has run, and a pass that has
|
|
4821
|
+
// run has read the window's records once already - so the read's total is two passes over them.
|
|
4822
|
+
readSymbolBytes + readPasses(symbolsToKeep !== null) * rowsToLoad * recordSize,
|
|
4823
|
+
this._symbolCache !== null
|
|
4411
4824
|
);
|
|
4412
4825
|
}
|
|
4413
|
-
warnLargeSymbolTable(
|
|
4826
|
+
warnLargeSymbolTable(
|
|
4827
|
+
symbolBytes,
|
|
4828
|
+
rowsToLoad,
|
|
4829
|
+
totalRows,
|
|
4830
|
+
fields.length,
|
|
4831
|
+
this._materialisesRows,
|
|
4832
|
+
this._symbolCache !== null
|
|
4833
|
+
);
|
|
4414
4834
|
for (const field of allFields) {
|
|
4415
4835
|
validateFieldMetadata(field, symbolTableSize, this._path);
|
|
4416
4836
|
}
|
|
@@ -4418,24 +4838,36 @@ var init_QvdFileReader = __esm({
|
|
|
4418
4838
|
const symbolTable = [];
|
|
4419
4839
|
for (const [position, field] of fields.entries()) {
|
|
4420
4840
|
this._throwIfAborted();
|
|
4841
|
+
const cached = this._symbolCache?.get(field["FieldName"]);
|
|
4842
|
+
if (cached) {
|
|
4843
|
+
symbolTable.push(cached);
|
|
4844
|
+
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
4845
|
+
continue;
|
|
4846
|
+
}
|
|
4421
4847
|
const area = await this._symbolAreaOf(field);
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
)
|
|
4848
|
+
const parsed = parseFieldSymbols(
|
|
4849
|
+
area.buffer,
|
|
4850
|
+
area.start,
|
|
4851
|
+
area.end,
|
|
4852
|
+
// Checked against the symbols the area holds, which is the one check that sees a terminator
|
|
4853
|
+
// damaged in the middle of it (#124). A cached field was checked when it was decoded, which is
|
|
4854
|
+
// why the cache may only hold a field a full walk produced.
|
|
4855
|
+
headerInteger(field["NoOfSymbols"]),
|
|
4856
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
4857
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
4858
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
4859
|
+
field["FieldName"],
|
|
4860
|
+
this._path,
|
|
4861
|
+
void 0,
|
|
4862
|
+
area.base
|
|
4438
4863
|
);
|
|
4864
|
+
symbolTable.push(parsed);
|
|
4865
|
+
if (this._symbolCache && symbolsToKeep === null) {
|
|
4866
|
+
if (this._symbolCache.size === 0) {
|
|
4867
|
+
this._cachedFor = this._fileFingerprint();
|
|
4868
|
+
}
|
|
4869
|
+
this._symbolCache.set(field["FieldName"], parsed);
|
|
4870
|
+
}
|
|
4439
4871
|
this._releaseSymbolArea(field);
|
|
4440
4872
|
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
4441
4873
|
}
|
|
@@ -4530,28 +4962,173 @@ var init_QvdFileReader = __esm({
|
|
|
4530
4962
|
await this._parseHeader();
|
|
4531
4963
|
this._emitProgress("header", 1, 1);
|
|
4532
4964
|
this._throwIfAborted();
|
|
4533
|
-
|
|
4534
|
-
const header = this._header["QvdTableHeader"];
|
|
4535
|
-
const columns = this._allFields.map((field) => field["FieldName"]);
|
|
4536
|
-
const rowCount = headerInteger(header["NoOfRecords"]);
|
|
4537
|
-
validateRecordCount(rowCount, this._path, "readMetadata");
|
|
4538
|
-
const shape = new exports.QvdDataFrame([], columns, header, {
|
|
4539
|
-
symbolTableBytes: headerInteger(header["Offset"]),
|
|
4540
|
-
totalRows: rowCount,
|
|
4541
|
-
rowsLoaded: 0,
|
|
4542
|
-
symbolFiltering: false,
|
|
4543
|
-
symbolsKept: null
|
|
4544
|
-
});
|
|
4545
|
-
return {
|
|
4546
|
-
columns,
|
|
4547
|
-
rowCount,
|
|
4548
|
-
columnCount: columns.length,
|
|
4549
|
-
fields: columns.map((name) => shape.getFieldMetadata(name)),
|
|
4550
|
-
fileMetadata: shape.fileMetadata,
|
|
4551
|
-
metadata: header
|
|
4552
|
-
};
|
|
4965
|
+
return this.describeParsed();
|
|
4553
4966
|
});
|
|
4554
4967
|
}
|
|
4968
|
+
/**
|
|
4969
|
+
* The schema and header of the file this reader has parsed, as `readMetadata()` reports them.
|
|
4970
|
+
*
|
|
4971
|
+
* Built from the parsed header and nothing else, so a caller holding a header - a `QvdFile` - can have
|
|
4972
|
+
* it without reading the file a second time.
|
|
4973
|
+
*
|
|
4974
|
+
* @return {any} The metadata.
|
|
4975
|
+
*/
|
|
4976
|
+
describeParsed() {
|
|
4977
|
+
assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
4978
|
+
const header = this._header["QvdTableHeader"];
|
|
4979
|
+
const columns = this._allFields.map((field) => field["FieldName"]);
|
|
4980
|
+
const rowCount = headerInteger(header["NoOfRecords"]);
|
|
4981
|
+
validateRecordCount(rowCount, this._path, "readMetadata");
|
|
4982
|
+
const shape = new exports.QvdDataFrame([], columns, header, {
|
|
4983
|
+
symbolTableBytes: headerInteger(header["Offset"]),
|
|
4984
|
+
totalRows: rowCount,
|
|
4985
|
+
rowsLoaded: 0,
|
|
4986
|
+
symbolFiltering: false,
|
|
4987
|
+
symbolsKept: null
|
|
4988
|
+
});
|
|
4989
|
+
return {
|
|
4990
|
+
columns,
|
|
4991
|
+
rowCount,
|
|
4992
|
+
columnCount: columns.length,
|
|
4993
|
+
fields: columns.map((name) => shape.getFieldMetadata(name)),
|
|
4994
|
+
fileMetadata: shape.fileMetadata,
|
|
4995
|
+
metadata: header
|
|
4996
|
+
};
|
|
4997
|
+
}
|
|
4998
|
+
/**
|
|
4999
|
+
* What a read of this file would cost, and whether it fits, without reading it.
|
|
5000
|
+
*
|
|
5001
|
+
* Reads the header and the file's size and nothing else, at the constant cost of `loadMetadata()`,
|
|
5002
|
+
* then asks the same question a read asks before it allocates anything - through the same function,
|
|
5003
|
+
* from the same numbers. That is the whole point: an answer computed a second way would be a second
|
|
5004
|
+
* opinion, and a read this approves would still be refused.
|
|
5005
|
+
*
|
|
5006
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [rawWindow]
|
|
5007
|
+
* The rows the read would cover, spelled any of the ways a read accepts.
|
|
5008
|
+
* @param {{chunkSize?: number|null}} [options] `chunkSize` when the read would be an `iterate()`,
|
|
5009
|
+
* which holds two chunks of rows rather than the window.
|
|
5010
|
+
* @return {Promise<any>} The answer - see `checkMemory`.
|
|
5011
|
+
*/
|
|
5012
|
+
async checkRead(rawWindow, { chunkSize = null } = {}) {
|
|
5013
|
+
const window = normaliseWindow(rawWindow, this._path);
|
|
5014
|
+
return await this._closingAfter(async () => {
|
|
5015
|
+
await this._parseHeaderChecked();
|
|
5016
|
+
return this.checkParsed(window, { chunkSize });
|
|
5017
|
+
});
|
|
5018
|
+
}
|
|
5019
|
+
/**
|
|
5020
|
+
* Reads this file's header, and nothing else, leaving it parsed on the reader.
|
|
5021
|
+
*
|
|
5022
|
+
* What `checkRead` and `QvdFile` both start with: the second asks many questions of one header, so the
|
|
5023
|
+
* read that produces it is separate from the questions. Every check a read makes before it trusts the
|
|
5024
|
+
* header's numbers is made here, so that nothing downstream has to wonder whether they hold.
|
|
5025
|
+
*
|
|
5026
|
+
* @return {Promise<void>} When the header is parsed and checked.
|
|
5027
|
+
*/
|
|
5028
|
+
async parseHeaderOnly() {
|
|
5029
|
+
return await this._closingAfter(async () => await this._parseHeaderChecked());
|
|
5030
|
+
}
|
|
5031
|
+
/**
|
|
5032
|
+
* `parseHeaderOnly`'s body, for a caller already inside a read session - `checkRead` is one.
|
|
5033
|
+
*
|
|
5034
|
+
* @return {Promise<void>} When the header is parsed and checked.
|
|
5035
|
+
* @private
|
|
5036
|
+
*/
|
|
5037
|
+
async _parseHeaderChecked() {
|
|
5038
|
+
await this._readData({ offset: 0, limit: null }, true);
|
|
5039
|
+
this._emitProgress("header", 0, 1);
|
|
5040
|
+
await this._parseHeader();
|
|
5041
|
+
this._emitProgress("header", 1, 1);
|
|
5042
|
+
this._throwIfAborted();
|
|
5043
|
+
assert4__default.default(
|
|
5044
|
+
this._header && this._selectedFields && this._allFields && this._symbolTableOffset !== null,
|
|
5045
|
+
"The QVD file header has not been parsed."
|
|
5046
|
+
);
|
|
5047
|
+
const header = this._header["QvdTableHeader"];
|
|
5048
|
+
const totalRows = headerInteger(header["NoOfRecords"]);
|
|
5049
|
+
const recordSize = headerInteger(header["RecordByteSize"]);
|
|
5050
|
+
const symbolTableLength = headerInteger(header["Offset"]);
|
|
5051
|
+
validateRecordSize(recordSize, this._path, "checkRead");
|
|
5052
|
+
validateRecordCount(totalRows, this._path, "checkRead");
|
|
5053
|
+
if (!this._headerMatchesFile) {
|
|
5054
|
+
throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
|
|
5055
|
+
file: this._path,
|
|
5056
|
+
fileSize: this._fileSize,
|
|
5057
|
+
requiredBytes: this._symbolTableOffset + symbolTableLength + totalRows * recordSize,
|
|
5058
|
+
stage: "checkRead"
|
|
5059
|
+
});
|
|
5060
|
+
}
|
|
5061
|
+
const tableLength = this._symbolTableLength();
|
|
5062
|
+
for (const field of this._allFields) {
|
|
5063
|
+
validateFieldMetadata(field, tableLength, this._path);
|
|
5064
|
+
validateFieldBitMetadata(field, recordSize, this._path);
|
|
5065
|
+
}
|
|
5066
|
+
validateSymbolAreas(this._allFields, this._path);
|
|
5067
|
+
}
|
|
5068
|
+
/**
|
|
5069
|
+
* What a read of the parsed header's file would cost, with no I/O at all.
|
|
5070
|
+
*
|
|
5071
|
+
* Separate from `checkRead` because a `QvdFile` asks this of one header many times - once per page a
|
|
5072
|
+
* viewer scrolls to - and the header is already in hand. `parseHeaderOnly` has to have run.
|
|
5073
|
+
*
|
|
5074
|
+
* @param {QvdRowWindow} window The rows the read would cover, normalised.
|
|
5075
|
+
* @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean}} [options]
|
|
5076
|
+
* `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask about a read other than the
|
|
5077
|
+
* one this reader was built for, which is what a `QvdFile` does per call.
|
|
5078
|
+
* @return {any} The answer - see `checkMemory`.
|
|
5079
|
+
*/
|
|
5080
|
+
checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0 } = {}) {
|
|
5081
|
+
assert4__default.default(this._header && this._allFields, "The QVD file header has not been parsed.");
|
|
5082
|
+
const header = this._header["QvdTableHeader"];
|
|
5083
|
+
const totalRows = headerInteger(header["NoOfRecords"]);
|
|
5084
|
+
const recordSize = headerInteger(header["RecordByteSize"]);
|
|
5085
|
+
const symbolTableLength = headerInteger(header["Offset"]);
|
|
5086
|
+
const builds = materialisesRows === void 0 ? this._materialisesRows : materialisesRows;
|
|
5087
|
+
const selected = selectFields(this._allFields, fields === void 0 ? this._requestedFields : fields, this._path);
|
|
5088
|
+
const resolved = resolveWindow(window, totalRows);
|
|
5089
|
+
const windowRows = resolved.limit;
|
|
5090
|
+
const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
|
|
5091
|
+
const analysisAhead = this._analysisAhead(window, resolved, totalRows, symbolTableLength);
|
|
5092
|
+
const measured = getMemoryBudget();
|
|
5093
|
+
const ask = /* @__PURE__ */ __name((asked, rows) => {
|
|
5094
|
+
const bytes = symbolBytesOf(this._fieldsHeldAfter(asked, this._allFields), symbolTableLength);
|
|
5095
|
+
const read = symbolBytesOf(this._fieldsReadBy(asked), symbolTableLength);
|
|
5096
|
+
return checkMemory({
|
|
5097
|
+
measured,
|
|
5098
|
+
symbolTableSize: bytes,
|
|
5099
|
+
maxRows: rows,
|
|
5100
|
+
totalRows,
|
|
5101
|
+
safetyFactor: this._memorySafetyFactor,
|
|
5102
|
+
columnCount: asked.length,
|
|
5103
|
+
materialisesRows: builds,
|
|
5104
|
+
live: liveRows,
|
|
5105
|
+
// A paging read keeps whole columns, so it is charged for whole columns - see `estimateMemoryUsage`.
|
|
5106
|
+
wholeSymbols: this._symbolCache !== null,
|
|
5107
|
+
bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead),
|
|
5108
|
+
// What it reads from the file, which is not what it holds: the symbol areas it has still to read,
|
|
5109
|
+
// and every record the window covers, read a slice at a time and not kept - twice over where the
|
|
5110
|
+
// symbol-usage pass will run, since it reads them before the decode reads them again.
|
|
5111
|
+
readBytes: read + readPasses(analysisAhead) * windowRows * recordSize
|
|
5112
|
+
});
|
|
5113
|
+
}, "ask");
|
|
5114
|
+
const answer = ask(selected, windowRows);
|
|
5115
|
+
if (!answer.fits && selected.length > 1) {
|
|
5116
|
+
const bySize = [...selected].sort(
|
|
5117
|
+
(a, b) => headerInteger(a["Length"]) - headerInteger(b["Length"])
|
|
5118
|
+
);
|
|
5119
|
+
for (let take = selected.length - 1; take >= 1; take -= 1) {
|
|
5120
|
+
const fewer = bySize.slice(0, take);
|
|
5121
|
+
if (ask(fewer, windowRows).fits) {
|
|
5122
|
+
answer.suggestions.push({
|
|
5123
|
+
option: "fields",
|
|
5124
|
+
value: fewer.map((field) => field["FieldName"])
|
|
5125
|
+
});
|
|
5126
|
+
break;
|
|
5127
|
+
}
|
|
5128
|
+
}
|
|
5129
|
+
}
|
|
5130
|
+
return answer;
|
|
5131
|
+
}
|
|
4555
5132
|
/**
|
|
4556
5133
|
* Loads the QVD file into memory and parses it.
|
|
4557
5134
|
*
|
|
@@ -4641,13 +5218,7 @@ var init_QvdFileReader = __esm({
|
|
|
4641
5218
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
|
|
4642
5219
|
*/
|
|
4643
5220
|
async *iterateRows(window, chunkSize) {
|
|
4644
|
-
|
|
4645
|
-
throw new exports.QvdValidationError("chunkSize must be a positive integer", {
|
|
4646
|
-
provided: chunkSize,
|
|
4647
|
-
type: typeof chunkSize,
|
|
4648
|
-
file: this._path
|
|
4649
|
-
});
|
|
4650
|
-
}
|
|
5221
|
+
requireChunkSize(chunkSize, this._path);
|
|
4651
5222
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
4652
5223
|
const rows = normaliseWindow(window, this._path);
|
|
4653
5224
|
this._startRead();
|
|
@@ -4722,7 +5293,7 @@ var init_QvdFileReader = __esm({
|
|
|
4722
5293
|
const rowsAvailable = resolved.limit;
|
|
4723
5294
|
let symbolsToKeep = null;
|
|
4724
5295
|
let symbolsKept = null;
|
|
4725
|
-
if (this.
|
|
5296
|
+
if (this._analysisAhead(window, resolved, totalRows, symbolTableLength)) {
|
|
4726
5297
|
symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
|
|
4727
5298
|
symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
|
|
4728
5299
|
}
|
|
@@ -4814,6 +5385,224 @@ var init_QvdFileReader = __esm({
|
|
|
4814
5385
|
}
|
|
4815
5386
|
});
|
|
4816
5387
|
|
|
5388
|
+
// src/QvdFile.js
|
|
5389
|
+
var QvdFile_exports = {};
|
|
5390
|
+
__export(QvdFile_exports, {
|
|
5391
|
+
QvdFile: () => exports.QvdFile
|
|
5392
|
+
});
|
|
5393
|
+
exports.QvdFile = void 0;
|
|
5394
|
+
var init_QvdFile = __esm({
|
|
5395
|
+
"src/QvdFile.js"() {
|
|
5396
|
+
init_QvdErrors();
|
|
5397
|
+
init_readOptions();
|
|
5398
|
+
exports.QvdFile = class {
|
|
5399
|
+
static {
|
|
5400
|
+
__name(this, "QvdFile");
|
|
5401
|
+
}
|
|
5402
|
+
/**
|
|
5403
|
+
* Not called directly - `QvdDataFrame.open()` is the way in, because a `QvdFile` is only ever a file
|
|
5404
|
+
* whose header has been read, and a constructor cannot wait for that.
|
|
5405
|
+
*
|
|
5406
|
+
* @param {any} reader The reader holding the parsed header.
|
|
5407
|
+
* @param {any} metadata What `readMetadata()` returns for this file.
|
|
5408
|
+
* @param {any} options The options the file was opened with.
|
|
5409
|
+
* @private
|
|
5410
|
+
*/
|
|
5411
|
+
constructor(reader, metadata, options) {
|
|
5412
|
+
this._reader = reader;
|
|
5413
|
+
this._metadata = metadata;
|
|
5414
|
+
this._options = options;
|
|
5415
|
+
this._closed = false;
|
|
5416
|
+
this._tail = Promise.resolve();
|
|
5417
|
+
this._readers = { rows: null, columns: null };
|
|
5418
|
+
}
|
|
5419
|
+
/**
|
|
5420
|
+
* The file's header and schema, as `QvdDataFrame.readMetadata()` returns them.
|
|
5421
|
+
*
|
|
5422
|
+
* Read when the file was opened, so this costs nothing and cannot fail.
|
|
5423
|
+
*
|
|
5424
|
+
* @return {any} The metadata.
|
|
5425
|
+
*/
|
|
5426
|
+
get metadata() {
|
|
5427
|
+
return this._metadata;
|
|
5428
|
+
}
|
|
5429
|
+
/**
|
|
5430
|
+
* Whether `close()` has been called.
|
|
5431
|
+
*
|
|
5432
|
+
* @return {boolean} True once it has.
|
|
5433
|
+
*/
|
|
5434
|
+
get closed() {
|
|
5435
|
+
return this._closed;
|
|
5436
|
+
}
|
|
5437
|
+
/**
|
|
5438
|
+
* What a read of this file would cost, and whether it fits - with no I/O at all.
|
|
5439
|
+
*
|
|
5440
|
+
* The same answer `QvdDataFrame.checkRead()` gives, from the header this file already holds, so a
|
|
5441
|
+
* viewer can size a page before asking for it without touching the disk.
|
|
5442
|
+
*
|
|
5443
|
+
* @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
|
|
5444
|
+
* as?: 'rows'|'columns', chunkSize?: number|null}} [options] The read being asked about - the same
|
|
5445
|
+
* bag `rows()` takes, plus `as` and `chunkSize` to say which shape of read it is.
|
|
5446
|
+
* @return {any} The answer - `fits`, `reason`, `estimate`, `budget`, `exact`, `suggestions`.
|
|
5447
|
+
* @throws {QvdValidationError} If the file is closed, or an option's value is not valid.
|
|
5448
|
+
*/
|
|
5449
|
+
check(options = {}) {
|
|
5450
|
+
this._refuseWhenClosed("check");
|
|
5451
|
+
const { as = "rows", chunkSize = null } = options;
|
|
5452
|
+
if (as !== "rows" && as !== "columns") {
|
|
5453
|
+
throw new exports.QvdValidationError("as must be 'rows' or 'columns'", {
|
|
5454
|
+
provided: as,
|
|
5455
|
+
reason: "option",
|
|
5456
|
+
option: "as",
|
|
5457
|
+
value: as,
|
|
5458
|
+
file: this._options.path
|
|
5459
|
+
});
|
|
5460
|
+
}
|
|
5461
|
+
if (chunkSize !== null) {
|
|
5462
|
+
requireChunkSize(chunkSize, this._options.path);
|
|
5463
|
+
}
|
|
5464
|
+
const reader = this._readers[as] ?? this._reader;
|
|
5465
|
+
return reader.checkParsed(normaliseWindow(windowFrom(options), this._options.path), {
|
|
5466
|
+
chunkSize,
|
|
5467
|
+
// Resolved here rather than left to the reader, exactly as `_page` resolves it. A warm reader is
|
|
5468
|
+
// still holding the last page's selection, and `checkParsed` falls back to it - so a `check()`
|
|
5469
|
+
// naming no fields answered for whatever the previous page happened to name. On a four-column
|
|
5470
|
+
// file after a page naming one of them, it reported 27,490 bytes for a read that costs 108,160:
|
|
5471
|
+
// understating, which is the direction that approves a read the read then refuses.
|
|
5472
|
+
fields: options.fields === void 0 ? this._options.fields ?? null : options.fields,
|
|
5473
|
+
materialisesRows: as === "rows"
|
|
5474
|
+
});
|
|
5475
|
+
}
|
|
5476
|
+
/**
|
|
5477
|
+
* Reads a page of rows.
|
|
5478
|
+
*
|
|
5479
|
+
* @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
|
|
5480
|
+
* onProgress?: Function, signal?: AbortSignal}} [options] The page, and how to read it. One bag,
|
|
5481
|
+
* as every other entry point takes: `offset` and `limit` say which rows, `fields` names a projection
|
|
5482
|
+
* for this page alone, and anything left out falls back to what the file was opened with.
|
|
5483
|
+
* @return {Promise<any>} The page, as a `QvdDataFrame`.
|
|
5484
|
+
* @throws {QvdValidationError} If the file is closed.
|
|
5485
|
+
*/
|
|
5486
|
+
async rows(options = {}) {
|
|
5487
|
+
this._refuseWhenClosed("rows");
|
|
5488
|
+
return await this._serialised(async () => await this._page(options, true, (reader, window) => reader.load(window)));
|
|
5489
|
+
}
|
|
5490
|
+
/**
|
|
5491
|
+
* Reads a page as columns, building no rows.
|
|
5492
|
+
*
|
|
5493
|
+
* @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
|
|
5494
|
+
* onProgress?: Function, signal?: AbortSignal}} [options] The page, as `rows()` takes it.
|
|
5495
|
+
* @return {Promise<any>} The page, as a `QvdColumnTable`.
|
|
5496
|
+
* @throws {QvdValidationError} If the file is closed.
|
|
5497
|
+
*/
|
|
5498
|
+
async columns(options = {}) {
|
|
5499
|
+
this._refuseWhenClosed("columns");
|
|
5500
|
+
return await this._serialised(
|
|
5501
|
+
async () => await this._page(options, false, (reader, window) => reader.loadColumnar(window))
|
|
5502
|
+
);
|
|
5503
|
+
}
|
|
5504
|
+
/**
|
|
5505
|
+
* Closes the file.
|
|
5506
|
+
*
|
|
5507
|
+
* Every call after it is refused with `reason: 'closed'`. Calling it twice is not an error: a
|
|
5508
|
+
* `finally` that closes and an `await using` that closes are both right, and both may run.
|
|
5509
|
+
*
|
|
5510
|
+
* @return {Promise<void>} When the pages already in flight have finished.
|
|
5511
|
+
*/
|
|
5512
|
+
async close() {
|
|
5513
|
+
if (this._closed) {
|
|
5514
|
+
return;
|
|
5515
|
+
}
|
|
5516
|
+
this._closed = true;
|
|
5517
|
+
await this._tail;
|
|
5518
|
+
for (const reader of Object.values(this._readers)) {
|
|
5519
|
+
reader?.endPaging();
|
|
5520
|
+
}
|
|
5521
|
+
this._readers = { rows: null, columns: null };
|
|
5522
|
+
this._reader.endPaging();
|
|
5523
|
+
this._reader = null;
|
|
5524
|
+
}
|
|
5525
|
+
/**
|
|
5526
|
+
* `await using` support, where the runtime has it.
|
|
5527
|
+
*
|
|
5528
|
+
* @return {Promise<void>} When closed.
|
|
5529
|
+
*/
|
|
5530
|
+
async [Symbol.asyncDispose]() {
|
|
5531
|
+
await this.close();
|
|
5532
|
+
}
|
|
5533
|
+
/**
|
|
5534
|
+
* Refuses a call on a closed file, in the vocabulary the rest of the API uses.
|
|
5535
|
+
*
|
|
5536
|
+
* @param {string} call The method the caller reached for, for the error.
|
|
5537
|
+
* @private
|
|
5538
|
+
*/
|
|
5539
|
+
_refuseWhenClosed(call) {
|
|
5540
|
+
if (this._closed) {
|
|
5541
|
+
throw new exports.QvdValidationError("The file is closed: open it again to read from it", {
|
|
5542
|
+
reason: "closed",
|
|
5543
|
+
call,
|
|
5544
|
+
file: this._options.path
|
|
5545
|
+
});
|
|
5546
|
+
}
|
|
5547
|
+
}
|
|
5548
|
+
/**
|
|
5549
|
+
* Runs `work` after everything asked for before it, and before everything asked for after.
|
|
5550
|
+
*
|
|
5551
|
+
* @param {() => Promise<any>} work The page to read.
|
|
5552
|
+
* @return {Promise<any>} Its result.
|
|
5553
|
+
* @private
|
|
5554
|
+
*/
|
|
5555
|
+
async _serialised(work) {
|
|
5556
|
+
const run = this._tail.then(work, work);
|
|
5557
|
+
this._tail = run.then(
|
|
5558
|
+
() => void 0,
|
|
5559
|
+
() => void 0
|
|
5560
|
+
);
|
|
5561
|
+
return await run;
|
|
5562
|
+
}
|
|
5563
|
+
/**
|
|
5564
|
+
* Reads one page, through the reader that keeps what the pages before it decoded.
|
|
5565
|
+
*
|
|
5566
|
+
* One reader for every page rather than one per page, which is what makes the symbol cache possible:
|
|
5567
|
+
* decoding the symbols is 72% of a page of a hundred rows from a 300,000-row file, and a reader built
|
|
5568
|
+
* fresh each time did all of it again. Two readers, because a columnar page builds no rows and a row
|
|
5569
|
+
* page does, and `materialisesRows` is fixed when a reader is constructed - so each shape keeps its
|
|
5570
|
+
* own, and its own cache.
|
|
5571
|
+
*
|
|
5572
|
+
* Options that belong to one call rather than to the file - the fields this page alone wants, and the
|
|
5573
|
+
* `onProgress` and `signal` watching it - are told to that shared reader for the next read and no
|
|
5574
|
+
* further. A page naming one of them used to build a reader of its own instead, which quietly turned
|
|
5575
|
+
* the cache off and the two-pass symbol path on: watching a page changed what the page did.
|
|
5576
|
+
*
|
|
5577
|
+
* @param {any} options What the call passed.
|
|
5578
|
+
* @param {boolean} builds Whether the page materialises rows.
|
|
5579
|
+
* @param {(reader: any, window: any) => Promise<any>} read The read to make.
|
|
5580
|
+
* @return {Promise<any>} The page.
|
|
5581
|
+
* @private
|
|
5582
|
+
*/
|
|
5583
|
+
async _page(options, builds, read) {
|
|
5584
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
5585
|
+
const kept = builds ? "rows" : "columns";
|
|
5586
|
+
if (!this._readers[kept]) {
|
|
5587
|
+
const reader2 = new QvdFileReader2(this._options.path, {
|
|
5588
|
+
...readerOptionsFrom(this._options),
|
|
5589
|
+
materialisesRows: builds
|
|
5590
|
+
});
|
|
5591
|
+
reader2.beginPaging();
|
|
5592
|
+
this._readers[kept] = reader2;
|
|
5593
|
+
}
|
|
5594
|
+
const reader = this._readers[kept];
|
|
5595
|
+
reader.selectForNextRead(options.fields === void 0 ? this._options.fields ?? null : options.fields);
|
|
5596
|
+
reader.observeNextRead({
|
|
5597
|
+
onProgress: options.onProgress ?? this._options.onProgress,
|
|
5598
|
+
signal: options.signal ?? this._options.signal
|
|
5599
|
+
});
|
|
5600
|
+
return await read(reader, windowFrom(options));
|
|
5601
|
+
}
|
|
5602
|
+
};
|
|
5603
|
+
}
|
|
5604
|
+
});
|
|
5605
|
+
|
|
4817
5606
|
// src/QvdDataFrame.js
|
|
4818
5607
|
function defaultFieldHeader(fieldName) {
|
|
4819
5608
|
return {
|
|
@@ -5562,6 +6351,111 @@ var init_QvdDataFrame = __esm({
|
|
|
5562
6351
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
5563
6352
|
return await new QvdFileReader2(path5, metadataOptionsFrom(options)).loadMetadata();
|
|
5564
6353
|
}
|
|
6354
|
+
/**
|
|
6355
|
+
* Answers what a read would cost, and whether it fits, without doing it.
|
|
6356
|
+
*
|
|
6357
|
+
* Takes the options `fromQvd()` takes, plus `as` and `chunkSize` to say which read is being asked
|
|
6358
|
+
* about. Reads the header and the file's size and nothing else, at a cost that does not grow with
|
|
6359
|
+
* the file.
|
|
6360
|
+
*
|
|
6361
|
+
* The answer comes from the same function a read consults before it allocates anything, from the
|
|
6362
|
+
* same numbers, so **a read this approves is not refused later for memory** - and every suggestion
|
|
6363
|
+
* it carries has been read back through the check, so following one gives a read that fits.
|
|
6364
|
+
*
|
|
6365
|
+
* It answers about resources, so it answers only for a header it can trust. A header whose numbers
|
|
6366
|
+
* are not usable, or that claims more than the file holds, is refused as a `QvdCorruptedError` rather
|
|
6367
|
+
* than answered: sizing a read from numbers the file contradicts produced a memory verdict about a
|
|
6368
|
+
* file whose real problem was structural, and it was wrong in both directions - approving a read the
|
|
6369
|
+
* library then refused, and refusing another with advice that was refused too.
|
|
6370
|
+
*
|
|
6371
|
+
* That covers everything a reader can tell from the header: a field area past the end of the symbol
|
|
6372
|
+
* table, two fields claiming one area, a `Bias` that is neither 0 nor -2, a `BitWidth` past 31. Damage
|
|
6373
|
+
* that is not in the header - a value or an index the file has spoiled - is still found only by
|
|
6374
|
+
* reading, and still refused as a `QvdCorruptedError` after this has said the read fits.
|
|
6375
|
+
*
|
|
6376
|
+
* ```js
|
|
6377
|
+
* const answer = await QvdDataFrame.checkRead('huge.qvd', {as: 'columns', fields: ['Amount']});
|
|
6378
|
+
*
|
|
6379
|
+
* if (!answer.fits) {
|
|
6380
|
+
* console.log(answer.reason); // 'memory'
|
|
6381
|
+
* console.log(answer.suggestions); // [{option: 'limit', value: 1250000}, ...]
|
|
6382
|
+
* }
|
|
6383
|
+
* ```
|
|
6384
|
+
*
|
|
6385
|
+
* @param {string} path The QVD file.
|
|
6386
|
+
* @param {object} [options] What `fromQvd()` takes, plus the two below.
|
|
6387
|
+
* @param {'rows'|'columns'} [options.as='rows'] Which read is being asked about: `rows` builds row
|
|
6388
|
+
* arrays and `columns` does not, which is most of what a read costs.
|
|
6389
|
+
* @param {number|null} [options.chunkSize=null] The chunk an `iterate()` would use, which holds two
|
|
6390
|
+
* chunks of rows rather than the whole window.
|
|
6391
|
+
* @return {Promise<any>} The answer: `fits`, `reason` when it does not, `estimate`, `budget`,
|
|
6392
|
+
* `exact` and `suggestions`.
|
|
6393
|
+
* @throws {QvdValidationError} If an option's value is not valid, with `context.reason` of `option`.
|
|
6394
|
+
* @throws {QvdCorruptedError} If the header cannot be read, its numbers are not usable, or it claims
|
|
6395
|
+
* more than the file holds. The read refuses such a file too, though it may name the fault
|
|
6396
|
+
* differently - it gets there by planning the index table, where this gets there from the size.
|
|
6397
|
+
*/
|
|
6398
|
+
static async checkRead(path5, options = {}) {
|
|
6399
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
6400
|
+
const { as = "rows", chunkSize = null } = options;
|
|
6401
|
+
if (as !== "rows" && as !== "columns") {
|
|
6402
|
+
throw new exports.QvdValidationError("as must be 'rows' or 'columns'", {
|
|
6403
|
+
provided: as,
|
|
6404
|
+
reason: "option",
|
|
6405
|
+
option: "as",
|
|
6406
|
+
value: as,
|
|
6407
|
+
file: path5
|
|
6408
|
+
});
|
|
6409
|
+
}
|
|
6410
|
+
if (chunkSize !== null) {
|
|
6411
|
+
requireChunkSize(chunkSize, path5);
|
|
6412
|
+
}
|
|
6413
|
+
const reader = new QvdFileReader2(path5, { ...readerOptionsFrom(options), materialisesRows: as === "rows" });
|
|
6414
|
+
return await reader.checkRead(windowFrom(options), { chunkSize });
|
|
6415
|
+
}
|
|
6416
|
+
/**
|
|
6417
|
+
* Opens a QVD file for paging, reading its header and nothing else.
|
|
6418
|
+
*
|
|
6419
|
+
* Every other entry point is one read from start to finish. A viewer showing a hundred rows at a time
|
|
6420
|
+
* pays the header again on every page, and `iterate()` goes forwards only - it cannot jump to row five
|
|
6421
|
+
* million and it cannot go back. This holds the header so that `check()` costs nothing and a page can
|
|
6422
|
+
* be asked for by position.
|
|
6423
|
+
*
|
|
6424
|
+
* ```js
|
|
6425
|
+
* const qvd = await QvdDataFrame.open('sales.qvd', {allowedDir: '/data'});
|
|
6426
|
+
*
|
|
6427
|
+
* qvd.metadata; // read once, when it opened
|
|
6428
|
+
* const answer = qvd.check({offset: 0, limit: 100}); // no I/O at all
|
|
6429
|
+
* const page = await qvd.rows({offset: 5_000_000, limit: 100});
|
|
6430
|
+
* const cols = await qvd.columns({offset: 0, limit: 100, fields: ['Amount']});
|
|
6431
|
+
*
|
|
6432
|
+
* await qvd.close();
|
|
6433
|
+
* ```
|
|
6434
|
+
*
|
|
6435
|
+
* The header is read once, and so is each column: a column decoded for one page is kept for the pages
|
|
6436
|
+
* after it, so the first page costs about what a single read costs and the ones after it are cheap.
|
|
6437
|
+
* What a file has decoded is charged to the memory check, so a page is refused rather than the process
|
|
6438
|
+
* aborting, and `close()` releases it - close a file you have finished with.
|
|
6439
|
+
*
|
|
6440
|
+
* Each page still opens the file, and a first touch still decodes a whole column rather than only as
|
|
6441
|
+
* far as the page needs.
|
|
6442
|
+
*
|
|
6443
|
+
* @param {string} path The QVD file.
|
|
6444
|
+
* @param {object} [options] What `fromQvd()` takes - `allowedDir`, `fields`, `duals`,
|
|
6445
|
+
* `coerceNumericStrings`, `memorySafetyFactor` - describing the file and how its values read. A
|
|
6446
|
+
* window means nothing here: pages carry their own.
|
|
6447
|
+
* @return {Promise<import('./QvdFile.js').QvdFile>} The open file.
|
|
6448
|
+
* @throws {QvdValidationError} If an option's value is not valid, with `context.reason` of `option`.
|
|
6449
|
+
* @throws {QvdCorruptedError} If the header cannot be read, or describes a file this is not.
|
|
6450
|
+
*/
|
|
6451
|
+
static async open(path5, options = {}) {
|
|
6452
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
6453
|
+
const { QvdFile: QvdFile2 } = await Promise.resolve().then(() => (init_QvdFile(), QvdFile_exports));
|
|
6454
|
+
const reader = new QvdFileReader2(path5, readerOptionsFrom(options));
|
|
6455
|
+
reader.beginPaging();
|
|
6456
|
+
await reader.parseHeaderOnly();
|
|
6457
|
+
return new QvdFile2(reader, reader.describeParsed(), { ...options, path: path5 });
|
|
6458
|
+
}
|
|
5565
6459
|
/**
|
|
5566
6460
|
* Constructs a data frame from a dictionary.
|
|
5567
6461
|
*
|
|
@@ -5866,6 +6760,7 @@ __name(dateToQlikSerial, "dateToQlikSerial");
|
|
|
5866
6760
|
// src/index.js
|
|
5867
6761
|
init_QvdDataFrame();
|
|
5868
6762
|
init_QvdColumnTable();
|
|
6763
|
+
init_QvdFile();
|
|
5869
6764
|
init_QvdFileReader();
|
|
5870
6765
|
init_QvdFileWriter();
|
|
5871
6766
|
init_QvdErrors();
|