qvdjs 0.10.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,9 +5,11 @@ import xml2 from 'xml2js';
5
5
  import assert2 from 'assert';
6
6
  import os from 'os';
7
7
  import v8 from 'v8';
8
+ import { types } from 'util';
8
9
 
9
10
  var __defProp = Object.defineProperty;
10
11
  var __getOwnPropNames = Object.getOwnPropertyNames;
12
+ var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
11
13
  var __esm = (fn, res) => function __init() {
12
14
  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
13
15
  };
@@ -17,10 +19,14 @@ var __export = (target, all) => {
17
19
  };
18
20
 
19
21
  // src/QvdErrors.js
20
- var QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
22
+ var ERROR_NAMES, QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
21
23
  var init_QvdErrors = __esm({
22
24
  "src/QvdErrors.js"() {
25
+ ERROR_NAMES = /* @__PURE__ */ new Map();
23
26
  QvdError = class extends Error {
27
+ static {
28
+ __name(this, "QvdError");
29
+ }
24
30
  /**
25
31
  * Constructs a new QVD error.
26
32
  *
@@ -30,13 +36,16 @@ var init_QvdErrors = __esm({
30
36
  */
31
37
  constructor(message, code, context = {}) {
32
38
  super(message);
33
- this.name = this.constructor.name;
39
+ this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
34
40
  this.code = code;
35
41
  this.context = context;
36
42
  Error.captureStackTrace(this, this.constructor);
37
43
  }
38
44
  };
39
45
  QvdParseError = class extends QvdError {
46
+ static {
47
+ __name(this, "QvdParseError");
48
+ }
40
49
  /**
41
50
  * Constructs a new QVD parse error.
42
51
  *
@@ -48,6 +57,9 @@ var init_QvdErrors = __esm({
48
57
  }
49
58
  };
50
59
  QvdValidationError = class extends QvdError {
60
+ static {
61
+ __name(this, "QvdValidationError");
62
+ }
51
63
  /**
52
64
  * Constructs a new QVD validation error.
53
65
  *
@@ -59,6 +71,9 @@ var init_QvdErrors = __esm({
59
71
  }
60
72
  };
61
73
  QvdIOError = class extends QvdError {
74
+ static {
75
+ __name(this, "QvdIOError");
76
+ }
62
77
  /**
63
78
  * Constructs a new QVD IO error.
64
79
  *
@@ -70,6 +85,9 @@ var init_QvdErrors = __esm({
70
85
  }
71
86
  };
72
87
  QvdCorruptedError = class extends QvdError {
88
+ static {
89
+ __name(this, "QvdCorruptedError");
90
+ }
73
91
  /**
74
92
  * Constructs a new QVD corrupted error.
75
93
  *
@@ -81,6 +99,9 @@ var init_QvdErrors = __esm({
81
99
  }
82
100
  };
83
101
  QvdSecurityError = class extends QvdError {
102
+ static {
103
+ __name(this, "QvdSecurityError");
104
+ }
84
105
  /**
85
106
  * Constructs a new QVD security error.
86
107
  *
@@ -91,163 +112,591 @@ var init_QvdErrors = __esm({
91
112
  super(message, "QVD_SECURITY_ERROR", context);
92
113
  }
93
114
  };
115
+ ERROR_NAMES.set(QvdError, "QvdError");
116
+ ERROR_NAMES.set(QvdParseError, "QvdParseError");
117
+ ERROR_NAMES.set(QvdValidationError, "QvdValidationError");
118
+ ERROR_NAMES.set(QvdIOError, "QvdIOError");
119
+ ERROR_NAMES.set(QvdCorruptedError, "QvdCorruptedError");
120
+ ERROR_NAMES.set(QvdSecurityError, "QvdSecurityError");
94
121
  }
95
122
  });
96
123
 
97
- // src/QvdSymbol.js
98
- var QvdSymbol;
99
- var init_QvdSymbol = __esm({
100
- "src/QvdSymbol.js"() {
124
+ // src/util/cellRules.js
125
+ function asDual(value) {
126
+ if (value === null || typeof value !== "object") {
127
+ return null;
128
+ }
129
+ try {
130
+ if (value[DUAL_BRAND] === true) {
131
+ return value;
132
+ }
133
+ if (!isPlainObject(value)) {
134
+ return null;
135
+ }
136
+ const keys = Object.keys(value);
137
+ return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
138
+ } catch {
139
+ return null;
140
+ }
141
+ }
142
+ function isPlainObject(value) {
143
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
144
+ return false;
145
+ }
146
+ const prototype = Object.getPrototypeOf(value);
147
+ return prototype === null || Object.getPrototypeOf(prototype) === null;
148
+ }
149
+ function isNumericText(text) {
150
+ return text.trim() !== "" && Number.isFinite(Number(text));
151
+ }
152
+ function isStoredAsInt(value) {
153
+ return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
154
+ }
155
+ function numberProblem(value) {
156
+ if (typeof value !== "number") {
157
+ return `is ${describeType(value)}, not a number`;
158
+ }
159
+ return Number.isFinite(value) ? null : `is ${String(value)}`;
160
+ }
161
+ function textProblem(value) {
162
+ if (typeof value !== "string") {
163
+ return { reason: "type" };
164
+ }
165
+ const position = value.indexOf(NUL);
166
+ if (position !== -1) {
167
+ return { reason: "nul", position };
168
+ }
169
+ return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
170
+ }
171
+ function unpairedSurrogateIndex(value) {
172
+ for (let index = 0; index < value.length; index++) {
173
+ const unit = value.charCodeAt(index);
174
+ if (unit < 55296 || unit > 57343) {
175
+ continue;
176
+ }
177
+ const next = value.charCodeAt(index + 1);
178
+ if (unit <= 56319 && next >= 56320 && next <= 57343) {
179
+ index++;
180
+ continue;
181
+ }
182
+ return index;
183
+ }
184
+ return -1;
185
+ }
186
+ function checkNumber(value, subject, context) {
187
+ if (numberProblem(value) === null) {
188
+ return;
189
+ }
190
+ if (subject === null && typeof value === "number") {
191
+ throw new QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
192
+ ...context,
193
+ provided: String(value)
194
+ });
195
+ }
196
+ throw new QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
197
+ ...context,
198
+ type: typeof value
199
+ });
200
+ }
201
+ function checkText(value, subject, context) {
202
+ const problem = textProblem(value);
203
+ if (problem === null) {
204
+ return;
205
+ }
206
+ if (problem.reason === "type") {
207
+ throw new QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
208
+ ...context,
209
+ type: typeof value
210
+ });
211
+ }
212
+ if (problem.reason === "surrogate") {
213
+ throw new QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
214
+ ...context,
215
+ position: problem.position
216
+ });
217
+ }
218
+ throw new QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
219
+ }
220
+ function constructorName(value) {
221
+ try {
222
+ const name = value.constructor?.name;
223
+ return typeof name === "string" && name !== "" ? name : null;
224
+ } catch {
225
+ return null;
226
+ }
227
+ }
228
+ function describeType(value) {
229
+ if (value === null) {
230
+ return "null";
231
+ }
232
+ if (typeof value === "number") {
233
+ return Number.isFinite(value) ? "a number" : String(value);
234
+ }
235
+ if (typeof value === "undefined") {
236
+ return "undefined";
237
+ }
238
+ if (typeof value !== "object") {
239
+ return `a ${typeof value}`;
240
+ }
241
+ let name;
242
+ try {
243
+ if (Array.isArray(value)) {
244
+ return "an array";
245
+ }
246
+ name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
247
+ if (name === "Object") {
248
+ if (!isPlainObject(value)) {
249
+ return "an object whose prototype is not Object.prototype";
250
+ }
251
+ const keys = Object.keys(value);
252
+ if (keys.length === 0) {
253
+ return "a plain object";
254
+ }
255
+ return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
256
+ }
257
+ } catch {
258
+ return "an object";
259
+ }
260
+ return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
261
+ }
262
+ var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
263
+ var init_cellRules = __esm({
264
+ "src/util/cellRules.js"() {
101
265
  init_QvdErrors();
102
- QvdSymbol = class _QvdSymbol {
103
- /**
104
- * Constructs a new QVD symbol.
105
- *
106
- * @param {number|null} intValue The integer value.
107
- * @param {number|null} doubleValue The double value.
108
- * @param {string|null} stringValue The string value.
109
- */
110
- constructor(intValue, doubleValue, stringValue) {
111
- this._intValue = intValue;
112
- this._doubleValue = doubleValue;
113
- this._stringValue = stringValue;
266
+ INT32_MIN = -2147483648;
267
+ INT32_MAX = 2147483647;
268
+ NUL = String.fromCharCode(0);
269
+ DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
270
+ __name(asDual, "asDual");
271
+ __name(isPlainObject, "isPlainObject");
272
+ __name(isNumericText, "isNumericText");
273
+ __name(isStoredAsInt, "isStoredAsInt");
274
+ __name(numberProblem, "numberProblem");
275
+ __name(textProblem, "textProblem");
276
+ __name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
277
+ __name(checkNumber, "checkNumber");
278
+ __name(checkText, "checkText");
279
+ __name(constructorName, "constructorName");
280
+ __name(describeType, "describeType");
281
+ }
282
+ });
283
+
284
+ // src/util/symbolBytes.js
285
+ function kindOf(number, text) {
286
+ if (number === null) {
287
+ return 4;
288
+ }
289
+ if (isStoredAsInt(number)) {
290
+ return text === null ? 1 : 5;
291
+ }
292
+ return text === null ? 2 : 6;
293
+ }
294
+ function symbolByteLength(kind, number, text) {
295
+ const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
296
+ const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
297
+ return 1 + numberBytes + textBytes;
298
+ }
299
+ function writeSymbol(buffer, offset, kind, number, text) {
300
+ buffer[offset++] = kind;
301
+ if (kind === 1 || kind === 5) {
302
+ offset = buffer.writeInt32LE(number, offset);
303
+ } else if (kind === 2 || kind === 6) {
304
+ offset = buffer.writeDoubleLE(number, offset);
305
+ }
306
+ if (kind >= 4) {
307
+ offset += buffer.write(text, offset, "utf8");
308
+ buffer[offset++] = 0;
309
+ }
310
+ return offset;
311
+ }
312
+ var init_symbolBytes = __esm({
313
+ "src/util/symbolBytes.js"() {
314
+ init_cellRules();
315
+ __name(kindOf, "kindOf");
316
+ __name(symbolByteLength, "symbolByteLength");
317
+ __name(writeSymbol, "writeSymbol");
318
+ }
319
+ });
320
+
321
+ // src/QvdDual.js
322
+ function defineHalves(target, number, text) {
323
+ Object.defineProperties(target, {
324
+ number: { value: number, enumerable: true },
325
+ text: { value: text, enumerable: true }
326
+ });
327
+ Object.freeze(target);
328
+ }
329
+ function dualFromSymbol(number, text) {
330
+ const dual = Object.create(QvdDual.prototype);
331
+ defineHalves(dual, number, text);
332
+ return dual;
333
+ }
334
+ var QvdDual;
335
+ var init_QvdDual = __esm({
336
+ "src/QvdDual.js"() {
337
+ init_cellRules();
338
+ QvdDual = class {
339
+ static {
340
+ __name(this, "QvdDual");
114
341
  }
115
342
  /**
116
- * Returns the integer value of this symbol.
343
+ * Constructs a dual value.
344
+ *
345
+ * The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
346
+ * integer inside the int32 range is stored as a dual int, anything else as a dual double.
117
347
  *
118
- * @return {number|null} The integer value.
348
+ * @param {number} number The numeric half. Must be a finite number.
349
+ * @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
350
+ * @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
119
351
  */
120
- get intValue() {
121
- return this._intValue;
352
+ constructor(number, text) {
353
+ checkNumber(number, "The number of a dual value", { half: "number" });
354
+ checkText(text, "The text of a dual value", { half: "text" });
355
+ defineHalves(this, number, text);
122
356
  }
123
357
  /**
124
- * Returns the double value of this symbol.
358
+ * Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
125
359
  *
126
- * @return {number|null} The double value.
360
+ * The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
361
+ * conversion ran, not which half the caller meant, and the fix is the same for all three.
362
+ *
363
+ * @return {never}
364
+ * @throws {TypeError} Always.
127
365
  */
128
- get doubleValue() {
129
- return this._doubleValue;
366
+ [Symbol.toPrimitive]() {
367
+ throw new TypeError(
368
+ `A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
369
+ );
130
370
  }
131
371
  /**
132
- * Returns the string value of this symbol.
372
+ * Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
133
373
  *
134
- * @return {string|null} The string value.
374
+ * @return {{number: number, text: string}} The dual as a plain object.
135
375
  */
136
- get stringValue() {
137
- return this._stringValue;
376
+ toJSON() {
377
+ return { number: this.number, text: this.text };
138
378
  }
139
379
  /**
140
- * Retrieves the primary value of this symbol. The primary value is descriptive raw value.
141
- * It is either the string value, the integer value or the double value, prioritized in this order.
380
+ * How Node's `util.inspect` and `console.log` show a dual.
142
381
  *
143
- * @return {number|string|null} The primary value.
382
+ * @return {string} For example `QvdDual(4.5, "4.50")`.
144
383
  */
145
- toPrimaryValue() {
146
- if (null != this._stringValue) {
147
- return this._stringValue;
148
- } else if (null != this._intValue) {
149
- return this._intValue;
150
- } else if (null != this._doubleValue) {
151
- return this._doubleValue;
152
- } else {
153
- return null;
154
- }
384
+ [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
385
+ return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
155
386
  }
156
- /**
157
- * Converts the symbol to its byte representation.
158
- *
159
- * @return {Buffer} The byte representation of the symbol.
160
- */
161
- toByteRepresentation() {
162
- if (this._intValue !== null && this._stringValue !== null) {
163
- const intBuffer = Buffer.alloc(4);
164
- intBuffer.writeInt32LE(this._intValue);
165
- const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
166
- return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
167
- } else if (this._doubleValue !== null && this._stringValue !== null) {
168
- const floatBuffer = Buffer.alloc(8);
169
- floatBuffer.writeDoubleLE(this._doubleValue);
170
- const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
171
- return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
172
- } else if (this._intValue !== null) {
173
- const buffer = Buffer.alloc(4);
174
- buffer.writeInt32LE(this._intValue);
175
- return Buffer.concat([Buffer.from([1]), buffer]);
176
- } else if (this._doubleValue !== null) {
177
- const buffer = Buffer.alloc(8);
178
- buffer.writeDoubleLE(this._doubleValue);
179
- return Buffer.concat([Buffer.from([2]), buffer]);
180
- } else if (this._stringValue !== null) {
181
- const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
182
- return Buffer.concat([Buffer.from([4]), buffer]);
183
- } else {
184
- throw new QvdValidationError("The symbol does not contain any value.", {
185
- intValue: this._intValue,
186
- doubleValue: this._doubleValue,
187
- stringValue: this._stringValue
188
- });
189
- }
387
+ /** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
388
+ get [Symbol.toStringTag]() {
389
+ return "QvdDual";
190
390
  }
191
391
  /**
192
- * Checks if this symbol is equal to another symbol.
392
+ * Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
393
+ * own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
193
394
  *
194
- * @param {*} value The object to compare with.
195
- * @return {boolean} True if the objects are equal, false otherwise.
196
- */
197
- equals(value) {
198
- if (!(value instanceof _QvdSymbol)) {
199
- return false;
200
- }
201
- return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
202
- }
203
- /**
204
- * Constructs a pure integer value symbol.
395
+ * Recognition only. The halves are checked when the value is written.
205
396
  *
206
- * @param {number} intValue The integer value.
207
- * @return {QvdSymbol} The constructed value symbol.
397
+ * @param {any} value Any value.
398
+ * @return {boolean} True for a dual cell.
208
399
  */
209
- static fromIntValue(intValue) {
210
- return new _QvdSymbol(intValue, null, null);
400
+ static isDual(value) {
401
+ return asDual(value) !== null;
211
402
  }
212
- /**
213
- * Constructs a pure double value symbol.
214
- *
215
- * @param {number} doubleValue The double value.
216
- * @return {QvdSymbol} The constructed value symbol.
217
- */
218
- static fromDoubleValue(doubleValue) {
219
- return new _QvdSymbol(null, doubleValue, null);
403
+ };
404
+ Object.defineProperty(QvdDual.prototype, DUAL_BRAND, { value: true });
405
+ __name(defineHalves, "defineHalves");
406
+ __name(dualFromSymbol, "dualFromSymbol");
407
+ }
408
+ });
409
+
410
+ // src/util/readOptions.js
411
+ function requireRowCount(value, name, filePath) {
412
+ if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
413
+ throw new QvdValidationError(`${name} must be a non-negative integer`, {
414
+ option: name,
415
+ provided: value,
416
+ type: typeof value,
417
+ file: filePath
418
+ });
419
+ }
420
+ return value;
421
+ }
422
+ function normaliseWindow(window, filePath) {
423
+ if (window === null || window === void 0) {
424
+ return { offset: 0, limit: null };
425
+ }
426
+ if (typeof window === "number") {
427
+ return { offset: 0, limit: requireRowCount(window, "maxRows", filePath) };
428
+ }
429
+ if (typeof window !== "object" || Array.isArray(window)) {
430
+ throw new QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
431
+ provided: window,
432
+ type: typeof window,
433
+ file: filePath
434
+ });
435
+ }
436
+ const { offset, limit, maxRows } = window;
437
+ const limitGiven = limit !== void 0 && limit !== null;
438
+ const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
439
+ if (limitGiven && maxRowsGiven) {
440
+ throw new QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
441
+ maxRows,
442
+ limit,
443
+ file: filePath
444
+ });
445
+ }
446
+ return {
447
+ offset: offset === void 0 || offset === null ? 0 : requireRowCount(offset, "offset", filePath),
448
+ limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
449
+ };
450
+ }
451
+ function resolveWindow(window, totalRows) {
452
+ const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
453
+ const offset = Math.min(window.offset, rows);
454
+ return {
455
+ offset,
456
+ limit: Math.max(0, Math.min(window.limit === null ? Infinity : window.limit, rows - offset))
457
+ };
458
+ }
459
+ function selectFields(fields, requested, filePath) {
460
+ if (requested === null || requested === void 0) {
461
+ return fields;
462
+ }
463
+ if (!Array.isArray(requested)) {
464
+ throw new QvdValidationError("fields must be an array of field names", {
465
+ provided: requested,
466
+ type: typeof requested,
467
+ file: filePath
468
+ });
469
+ }
470
+ const available = fields.map((field) => field["FieldName"]);
471
+ if (requested.length === 0) {
472
+ throw new QvdValidationError("fields must name at least one field", {
473
+ availableColumns: available,
474
+ file: filePath
475
+ });
476
+ }
477
+ const seen = /* @__PURE__ */ new Set();
478
+ return requested.map((name) => {
479
+ if (typeof name !== "string") {
480
+ throw new QvdValidationError("Field names must be strings", {
481
+ provided: name,
482
+ type: typeof name,
483
+ availableColumns: available,
484
+ file: filePath
485
+ });
486
+ }
487
+ if (seen.has(name)) {
488
+ throw new QvdValidationError(`Field '${name}' is listed twice`, {
489
+ column: name,
490
+ fields: requested,
491
+ file: filePath
492
+ });
493
+ }
494
+ seen.add(name);
495
+ const index = available.indexOf(name);
496
+ if (index === -1) {
497
+ throw new QvdValidationError(`Column '${name}' does not exist`, {
498
+ column: name,
499
+ availableColumns: available,
500
+ file: filePath
501
+ });
502
+ }
503
+ return fields[index];
504
+ });
505
+ }
506
+ function normaliseDuals(value, filePath) {
507
+ if (value === void 0 || value === null) {
508
+ return "number";
509
+ }
510
+ if (!DUAL_MODES.includes(value)) {
511
+ throw new QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
512
+ option: "duals",
513
+ provided: value,
514
+ file: filePath
515
+ });
516
+ }
517
+ return value;
518
+ }
519
+ function normaliseCoerceNumericStrings(value, filePath) {
520
+ if (value === void 0 || value === null) {
521
+ return false;
522
+ }
523
+ if (typeof value !== "boolean") {
524
+ throw new QvdValidationError("coerceNumericStrings must be true or false", {
525
+ option: "coerceNumericStrings",
526
+ provided: value,
527
+ type: typeof value,
528
+ file: filePath
529
+ });
530
+ }
531
+ return value;
532
+ }
533
+ function readerOptionsFrom(options) {
534
+ return {
535
+ allowedDir: options.allowedDir,
536
+ memorySafetyFactor: options.memorySafetyFactor,
537
+ symbolFilteringThreshold: options.symbolFilteringThreshold,
538
+ fields: options.fields === void 0 ? null : options.fields,
539
+ duals: options.duals,
540
+ coerceNumericStrings: options.coerceNumericStrings,
541
+ onProgress: options.onProgress,
542
+ signal: options.signal
543
+ };
544
+ }
545
+ function metadataOptionsFrom(options) {
546
+ return {
547
+ allowedDir: options.allowedDir,
548
+ onProgress: options.onProgress,
549
+ signal: options.signal
550
+ };
551
+ }
552
+ function windowFrom(options) {
553
+ return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
554
+ }
555
+ var DUAL_MODES;
556
+ var init_readOptions = __esm({
557
+ "src/util/readOptions.js"() {
558
+ init_QvdErrors();
559
+ __name(requireRowCount, "requireRowCount");
560
+ __name(normaliseWindow, "normaliseWindow");
561
+ __name(resolveWindow, "resolveWindow");
562
+ __name(selectFields, "selectFields");
563
+ DUAL_MODES = Object.freeze(["number", "text", "both"]);
564
+ __name(normaliseDuals, "normaliseDuals");
565
+ __name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
566
+ __name(readerOptionsFrom, "readerOptionsFrom");
567
+ __name(metadataOptionsFrom, "metadataOptionsFrom");
568
+ __name(windowFrom, "windowFrom");
569
+ }
570
+ });
571
+
572
+ // src/util/storedSymbols.js
573
+ function trustStoredSymbols(entries) {
574
+ const record = Object.freeze(entries);
575
+ trusted.add(record);
576
+ return record;
577
+ }
578
+ function attachStoredSymbols(metadata, record) {
579
+ if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
580
+ Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
581
+ }
582
+ }
583
+ function refuse(message, context) {
584
+ throw new QvdValidationError(message, context);
585
+ }
586
+ function normaliseStoredSymbols(record) {
587
+ if (record === null || record === void 0) {
588
+ return null;
589
+ }
590
+ if (typeof record === "object" && trusted.has(record)) {
591
+ return record;
592
+ }
593
+ if (!Array.isArray(record)) {
594
+ refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
595
+ option: "storedSymbols",
596
+ type: typeof record
597
+ });
598
+ }
599
+ const fields = /* @__PURE__ */ new Set();
600
+ const entries = record.map((entry, index) => {
601
+ if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
602
+ refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
603
+ }
604
+ const { field, values, numbers, texts } = entry;
605
+ if (fields.has(field)) {
606
+ refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
607
+ }
608
+ fields.add(field);
609
+ if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
610
+ refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
611
+ field,
612
+ entry: index
613
+ });
614
+ }
615
+ for (let symbol = 0; symbol < values.length; symbol++) {
616
+ const value = values[symbol];
617
+ const number = numbers[symbol];
618
+ const text = texts[symbol];
619
+ const context = { field, symbol };
620
+ if (typeof value !== "number" && typeof value !== "string") {
621
+ refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
622
+ ...context,
623
+ type: typeof value
624
+ });
220
625
  }
221
- /**
222
- * Constructs a pure string value symbol.
223
- *
224
- * @param {string} stringValue The string value.
225
- * @return {QvdSymbol} The constructed value symbol.
226
- */
227
- static fromStringValue(stringValue) {
228
- return new _QvdSymbol(null, null, stringValue);
626
+ if (number !== null) {
627
+ checkNumber(number, "A stored symbol's number", context);
229
628
  }
230
- /**
231
- * Constructs a dual value symbol from an integer and a string value.
232
- *
233
- * @param {number} intValue The integer value.
234
- * @param {string} stringValue The string value.
235
- * @return {QvdSymbol} The constructed value symbol.
236
- */
237
- static fromDualIntValue(intValue, stringValue) {
238
- return new _QvdSymbol(intValue, null, stringValue);
629
+ if (text !== null) {
630
+ checkText(text, "A stored symbol's text", context);
239
631
  }
240
- /**
241
- * Constructs a dual value symbol from a double and a string value.
242
- *
243
- * @param {number} doubleValue The double value.
244
- * @param {string} stringValue The string value.
245
- * @return {QvdSymbol} The constructed value symbol.
246
- */
247
- static fromDualDoubleValue(doubleValue, stringValue) {
248
- return new _QvdSymbol(null, doubleValue, stringValue);
632
+ const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
633
+ if (!consistent) {
634
+ refuse(
635
+ number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
636
+ { ...context, value, number, text }
637
+ );
249
638
  }
250
- };
639
+ }
640
+ return Object.freeze({
641
+ field,
642
+ values: Object.freeze(values.slice()),
643
+ numbers: Object.freeze(numbers.slice()),
644
+ texts: Object.freeze(texts.slice())
645
+ });
646
+ });
647
+ return trustStoredSymbols(entries);
648
+ }
649
+ function narrowStoredSymbols(record, columns) {
650
+ if (record === null) {
651
+ return null;
652
+ }
653
+ const kept = record.filter((entry) => columns.includes(entry.field));
654
+ return kept.length === record.length ? record : trustStoredSymbols(kept);
655
+ }
656
+ function storedSymbolsEntry(record, field) {
657
+ if (record === null) {
658
+ return null;
659
+ }
660
+ return record.find((entry) => entry.field === field) ?? null;
661
+ }
662
+ function firstTextByValue(entry) {
663
+ const byValue = /* @__PURE__ */ new Map();
664
+ for (let index = entry.values.length - 1; index >= 0; index--) {
665
+ const text = entry.texts[index];
666
+ if (text !== null) {
667
+ byValue.set(entry.values[index], text);
668
+ }
669
+ }
670
+ return byValue;
671
+ }
672
+ function storedTextOf(entry, value) {
673
+ let byValue = firstTexts.get(entry);
674
+ if (byValue === void 0) {
675
+ byValue = firstTextByValue(entry);
676
+ firstTexts.set(entry, byValue);
677
+ }
678
+ return byValue.get(value) ?? null;
679
+ }
680
+ function sameValueZero(a, b) {
681
+ return a === b || a !== a && b !== b;
682
+ }
683
+ var STORED_SYMBOLS, trusted, firstTexts;
684
+ var init_storedSymbols = __esm({
685
+ "src/util/storedSymbols.js"() {
686
+ init_QvdErrors();
687
+ init_cellRules();
688
+ STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
689
+ trusted = /* @__PURE__ */ new WeakSet();
690
+ __name(trustStoredSymbols, "trustStoredSymbols");
691
+ __name(attachStoredSymbols, "attachStoredSymbols");
692
+ __name(refuse, "refuse");
693
+ __name(normaliseStoredSymbols, "normaliseStoredSymbols");
694
+ __name(narrowStoredSymbols, "narrowStoredSymbols");
695
+ __name(storedSymbolsEntry, "storedSymbolsEntry");
696
+ firstTexts = /* @__PURE__ */ new WeakMap();
697
+ __name(firstTextByValue, "firstTextByValue");
698
+ __name(storedTextOf, "storedTextOf");
699
+ __name(sameValueZero, "sameValueZero");
251
700
  }
252
701
  });
253
702
  function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
@@ -352,6 +801,10 @@ function validatePath(filePath, allowedDir) {
352
801
  var init_validatePath = __esm({
353
802
  "src/util/validatePath.js"() {
354
803
  init_QvdErrors();
804
+ __name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
805
+ __name(resolveDeepestExisting, "resolveDeepestExisting");
806
+ __name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
807
+ __name(validatePath, "validatePath");
355
808
  }
356
809
  });
357
810
 
@@ -401,6 +854,9 @@ var init_bitUtils = __esm({
401
854
  "src/util/bitUtils.js"() {
402
855
  MAX_BIT_WIDTH = 31;
403
856
  POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
857
+ __name(fieldGeometry, "fieldGeometry");
858
+ __name(decodeIndexColumn, "decodeIndexColumn");
859
+ __name(writeBitField, "writeBitField");
404
860
  }
405
861
  });
406
862
 
@@ -409,14 +865,310 @@ var QvdFileWriter_exports = {};
409
865
  __export(QvdFileWriter_exports, {
410
866
  QvdFileWriter: () => QvdFileWriter
411
867
  });
412
- var QvdFileWriter;
868
+ function notXmlIndex(value) {
869
+ for (let index = 0; index < value.length; index++) {
870
+ const unit = value.charCodeAt(index);
871
+ if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
872
+ return index;
873
+ }
874
+ }
875
+ return -1;
876
+ }
877
+ function checkHeaderText(value, subject, context) {
878
+ checkText(value, subject, context);
879
+ const position = notXmlIndex(value);
880
+ if (position !== -1) {
881
+ const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
882
+ throw new QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
883
+ }
884
+ }
885
+ function checkHeaderTexts(value, property, owner, context) {
886
+ if (typeof value === "string") {
887
+ checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
888
+ } else if (Array.isArray(value)) {
889
+ value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
890
+ } else if (value !== null && typeof value === "object") {
891
+ for (const [key, item] of Object.entries(value)) {
892
+ checkHeaderTexts(item, `${property}.${key}`, owner, context);
893
+ }
894
+ }
895
+ }
896
+ function validateColumnNames(columns, filePath) {
897
+ const seen = /* @__PURE__ */ new Set();
898
+ columns.forEach((name, index) => {
899
+ if (typeof name !== "string" || name.length === 0) {
900
+ throw new QvdValidationError("Field names must be non-empty strings", {
901
+ column: index,
902
+ provided: name,
903
+ type: typeof name,
904
+ file: filePath,
905
+ stage: "buildSymbolTable"
906
+ });
907
+ }
908
+ checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
909
+ if (seen.has(name)) {
910
+ throw new QvdValidationError(`Field '${name}' appears twice`, {
911
+ column: name,
912
+ columnIndex: index,
913
+ file: filePath,
914
+ stage: "buildSymbolTable"
915
+ });
916
+ }
917
+ seen.add(name);
918
+ });
919
+ }
920
+ function refuseCell(value, column, row, filePath) {
921
+ let resemblesDual = false;
922
+ try {
923
+ resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
924
+ } catch {
925
+ }
926
+ throw new QvdValidationError(
927
+ `A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
928
+ {
929
+ column,
930
+ row,
931
+ type: typeof value,
932
+ constructor: constructorName(value) ?? void 0,
933
+ file: filePath,
934
+ stage: "buildSymbolTable"
935
+ }
936
+ );
937
+ }
938
+ function valuesAreNumbers(entry) {
939
+ const { values, numbers } = entry;
940
+ for (let index = 0; index < values.length; index++) {
941
+ if (!sameValueZero(values[index], numbers[index])) {
942
+ return false;
943
+ }
944
+ }
945
+ return true;
946
+ }
947
+ function newSlot(column, key, text) {
948
+ const slot = column.keys.length;
949
+ column.keys.push(key);
950
+ column.texts.push(text);
951
+ if (typeof key === "number") {
952
+ column.facts.hasNumber = true;
953
+ if (!Number.isInteger(key)) column.facts.hasFraction = true;
954
+ } else {
955
+ column.facts.hasNonNumber = true;
956
+ }
957
+ return slot;
958
+ }
959
+ function slotFor(column, key, text) {
960
+ let slot = column.byKey.get(key);
961
+ if (slot === void 0) {
962
+ slot = newSlot(column, key, text);
963
+ column.byKey.set(key, slot);
964
+ } else if (text !== null && column.texts[slot] === null && typeof key === "number") {
965
+ column.texts[slot] = text;
966
+ }
967
+ return slot;
968
+ }
969
+ function standsForSeveral(indices, numbers, texts) {
970
+ let number = null;
971
+ let string = null;
972
+ for (const index of indices) {
973
+ if (numbers[index] !== null) {
974
+ if (number === null) {
975
+ number = numbers[index];
976
+ } else if (!sameValueZero(number, numbers[index])) {
977
+ return true;
978
+ }
979
+ } else if (string === null) {
980
+ string = texts[index];
981
+ } else if (string !== texts[index]) {
982
+ return true;
983
+ }
984
+ }
985
+ return number !== null && string !== null;
986
+ }
987
+ function ambiguousAsUncoercedText(entry) {
988
+ const { values, numbers, texts } = entry;
989
+ for (let index = 0; index < values.length; index++) {
990
+ if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
991
+ if (!isNumericText(texts[index])) {
992
+ return false;
993
+ }
994
+ }
995
+ }
996
+ const byShown = /* @__PURE__ */ new Map();
997
+ for (let index = 0; index < values.length; index++) {
998
+ const shown = texts[index] ?? numbers[index];
999
+ const indices = byShown.get(shown);
1000
+ if (indices === void 0) {
1001
+ byShown.set(shown, [index]);
1002
+ } else {
1003
+ indices.push(index);
1004
+ }
1005
+ }
1006
+ for (const indices of byShown.values()) {
1007
+ if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
1008
+ return true;
1009
+ }
1010
+ }
1011
+ return false;
1012
+ }
1013
+ function unambiguousWithoutDuals(column) {
1014
+ const { numbers, texts } = column.entry;
1015
+ for (const found of column.byValue.values()) {
1016
+ if (typeof found !== "number") {
1017
+ const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
1018
+ if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
1019
+ return false;
1020
+ }
1021
+ }
1022
+ }
1023
+ return true;
1024
+ }
1025
+ function ambiguityRemedy(column) {
1026
+ const { values, numbers, texts } = column.entry;
1027
+ let coerced = false;
1028
+ let other = false;
1029
+ for (const [value, found] of column.byValue) {
1030
+ if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
1031
+ if (typeof value === "number") {
1032
+ coerced = true;
1033
+ } else {
1034
+ other = true;
1035
+ }
1036
+ }
1037
+ }
1038
+ if (!coerced) {
1039
+ return "Read the field with {duals: 'both'}, or write QvdDual cells.";
1040
+ }
1041
+ if (!other && !ambiguousAsUncoercedText(column.entry)) {
1042
+ return "Read the field without {coerceNumericStrings: true}.";
1043
+ }
1044
+ if (unambiguousWithoutDuals(column)) {
1045
+ return "Read the field with {duals: 'both'}.";
1046
+ }
1047
+ return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
1048
+ }
1049
+ function slotForCell(column, value, row, filePath) {
1050
+ if (column.byCell === column.byKey) {
1051
+ return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
1052
+ }
1053
+ const found = column.byValue.get(value);
1054
+ if (found === void 0) {
1055
+ return slotFor(column, value, null);
1056
+ }
1057
+ const { numbers, texts } = column.entry;
1058
+ const indices = typeof found === "number" ? [found] : found;
1059
+ if (standsForSeveral(indices, numbers, texts)) {
1060
+ const stored = [];
1061
+ for (const index of indices) {
1062
+ if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
1063
+ stored.push({ number: numbers[index], text: texts[index] });
1064
+ }
1065
+ }
1066
+ throw new QvdValidationError(
1067
+ `The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
1068
+ { column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
1069
+ );
1070
+ }
1071
+ let number = null;
1072
+ let numberText = null;
1073
+ for (const index of indices) {
1074
+ if (numbers[index] === null) {
1075
+ return slotFor(column, texts[index], null);
1076
+ }
1077
+ number ??= numbers[index];
1078
+ numberText ??= texts[index];
1079
+ }
1080
+ return slotFor(column, number, numberText);
1081
+ }
1082
+ function slotForObject(column, value, row, filePath) {
1083
+ const dual = asDual(value);
1084
+ if (dual === null) {
1085
+ refuseCell(value, column.name, row, filePath);
1086
+ }
1087
+ const { number, text } = dual;
1088
+ if (numberProblem(number) !== null) {
1089
+ checkNumber(number, "The number of a dual value", {
1090
+ column: column.name,
1091
+ row,
1092
+ half: "number",
1093
+ file: filePath,
1094
+ stage: "buildSymbolTable"
1095
+ });
1096
+ }
1097
+ if (textProblem(text) !== null) {
1098
+ checkText(text, "The text of a dual value", {
1099
+ column: column.name,
1100
+ row,
1101
+ half: "text",
1102
+ file: filePath,
1103
+ stage: "buildSymbolTable"
1104
+ });
1105
+ }
1106
+ const slot = slotFor(column, number, text);
1107
+ if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
1108
+ column.byObject.set(value, slot);
1109
+ }
1110
+ return slot;
1111
+ }
1112
+ function pruneContradictedTags(tags, facts) {
1113
+ if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
1114
+ return tags || {};
1115
+ }
1116
+ const list = Array.isArray(tags.String) ? tags.String : [tags.String];
1117
+ const kept = list.filter((tag) => {
1118
+ if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
1119
+ if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
1120
+ if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
1121
+ return true;
1122
+ });
1123
+ if (kept.length === list.length) {
1124
+ return tags;
1125
+ }
1126
+ return kept.length === 0 ? {} : { ...tags, String: kept };
1127
+ }
1128
+ function resetContradictedNumberFormat(numberFormat, facts) {
1129
+ if (!numberFormat) {
1130
+ return { ...UNKNOWN_NUMBER_FORMAT };
1131
+ }
1132
+ if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
1133
+ return { ...UNKNOWN_NUMBER_FORMAT };
1134
+ }
1135
+ return numberFormat;
1136
+ }
1137
+ var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, QvdFileWriter;
413
1138
  var init_QvdFileWriter = __esm({
414
1139
  "src/QvdFileWriter.js"() {
415
- init_QvdSymbol();
416
1140
  init_QvdErrors();
417
1141
  init_validatePath();
418
1142
  init_bitUtils();
419
- QvdFileWriter = class _QvdFileWriter {
1143
+ init_cellRules();
1144
+ init_symbolBytes();
1145
+ init_storedSymbols();
1146
+ __name(notXmlIndex, "notXmlIndex");
1147
+ __name(checkHeaderText, "checkHeaderText");
1148
+ __name(checkHeaderTexts, "checkHeaderTexts");
1149
+ __name(validateColumnNames, "validateColumnNames");
1150
+ __name(refuseCell, "refuseCell");
1151
+ OBJECT_MEMO_BASE = 64;
1152
+ __name(valuesAreNumbers, "valuesAreNumbers");
1153
+ __name(newSlot, "newSlot");
1154
+ __name(slotFor, "slotFor");
1155
+ __name(standsForSeveral, "standsForSeveral");
1156
+ __name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
1157
+ __name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
1158
+ __name(ambiguityRemedy, "ambiguityRemedy");
1159
+ __name(slotForCell, "slotForCell");
1160
+ __name(slotForObject, "slotForObject");
1161
+ NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
1162
+ TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
1163
+ WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
1164
+ NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
1165
+ UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
1166
+ __name(pruneContradictedTags, "pruneContradictedTags");
1167
+ __name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
1168
+ QvdFileWriter = class {
1169
+ static {
1170
+ __name(this, "QvdFileWriter");
1171
+ }
420
1172
  /**
421
1173
  * Constructs a new QVD file writer.
422
1174
  *
@@ -437,7 +1189,8 @@ var init_QvdFileWriter = __esm({
437
1189
  this._onProgress = onProgress;
438
1190
  this._header = null;
439
1191
  this._symbolBuffer = null;
440
- this._symbolTable = null;
1192
+ this._symbolCounts = null;
1193
+ this._symbolFacts = null;
441
1194
  this._symbolTableMetadata = null;
442
1195
  this._indexBuffer = null;
443
1196
  this._symbolIndexByValue = null;
@@ -548,19 +1301,12 @@ var init_QvdFileWriter = __esm({
548
1301
  BitOffset: this._indexTableMetadata?.[index][0],
549
1302
  BitWidth: this._indexTableMetadata?.[index][1],
550
1303
  Bias: this._indexTableMetadata?.[index][2],
551
- NoOfSymbols: this._symbolTable?.[index].length,
1304
+ NoOfSymbols: this._symbolCounts?.[index],
552
1305
  Offset: this._symbolTableMetadata?.[index][0],
553
1306
  Length: this._symbolTableMetadata?.[index][1],
554
1307
  Comment: existingField?.Comment || "",
555
- NumberFormat: existingField?.NumberFormat || {
556
- Type: "UNKNOWN",
557
- nDec: "0",
558
- UseThou: "0",
559
- Fmt: "",
560
- Dec: "",
561
- Thou: ""
562
- },
563
- Tags: existingField?.Tags || {}
1308
+ NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
1309
+ Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
564
1310
  };
565
1311
  })
566
1312
  },
@@ -570,6 +1316,19 @@ var init_QvdFileWriter = __esm({
570
1316
  Length: this._indexBuffer?.length
571
1317
  }
572
1318
  };
1319
+ const { Fields, ...table } = xmlObject.QvdTableHeader;
1320
+ for (const [property, value] of Object.entries(table)) {
1321
+ checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
1322
+ }
1323
+ for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
1324
+ for (const [property, value] of Object.entries(field)) {
1325
+ checkHeaderTexts(value, property, `field '${FieldName}'`, {
1326
+ column: FieldName,
1327
+ file: this._path,
1328
+ stage: "buildHeader"
1329
+ });
1330
+ }
1331
+ }
573
1332
  const builder = new xml2.Builder({
574
1333
  renderOpts: {
575
1334
  pretty: true,
@@ -583,25 +1342,49 @@ var init_QvdFileWriter = __esm({
583
1342
  /**
584
1343
  * Builds the symbol table of the QVD file.
585
1344
  *
586
- * PERFORMANCE OPTIMIZATION: This method uses a single-pass algorithm to build
587
- * symbol tables for all columns simultaneously. This reduces time complexity from
588
- * O(n×m×s) to O(n×m) where n=rows, m=columns, s=symbols per column.
1345
+ * One pass over the rows finds each column's distinct values in the order they first appear,
1346
+ * which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
1347
+ * column encodes them: every symbol is sized first, then written into one buffer of exactly that
1348
+ * size.
1349
+ *
1350
+ * What each value is stored as:
1351
+ *
1352
+ * | Cell | Symbol |
1353
+ * | --- | --- |
1354
+ * | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
1355
+ * | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
1356
+ * | any other finite number | pure double, type 2 |
1357
+ * | a string | pure string, type 4 |
1358
+ * | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
1359
+ * | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
1360
+ *
1361
+ * A number is a pure number, with no text. It used to be written as a dual whose text was
1362
+ * `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
1363
+ * file Qlik wrote with pure numbers came back out of a read and a write with every one of them
1364
+ * turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
1365
+ * A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
1366
+ * UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
1367
+ * an unpaired surrogate has no UTF-8 encoding at all.
1368
+ *
1369
+ * A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
1370
+ * symbol with the first text in row order, and a plain number with the same number as a dual joins
1371
+ * the dual, in either order - so no text a caller supplied is lost to a plain number that came
1372
+ * first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
589
1373
  *
590
- * Algorithm:
591
- * 1. Initialize a Set for each column to collect unique values
592
- * 2. Single pass through all data rows, adding values to corresponding Sets
593
- * 3. Convert Sets to arrays and create QvdSymbol instances
594
- * 4. Serialize symbols to binary format and update metadata
1374
+ * A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
1375
+ * string read as a number. Its record says what each such cell stands for, so the frame writes back
1376
+ * the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
1377
+ * rows the cells were moved to. A cell the record maps to more than one stored value is refused.
595
1378
  *
596
- * This approach provides:
597
- * - 80-90% performance improvement for large datasets (100K+ rows)
598
- * - Better cache locality (process all columns in one data traversal)
599
- * - Lower memory pressure (no intermediate arrays per column)
1379
+ * The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
1380
+ * for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
1381
+ * repository; `benchmarks/` measures what the writer costs now, which is the useful number.
600
1382
  *
601
1383
  * @private
602
1384
  */
603
1385
  _buildSymbolTable() {
604
- this._symbolTable = [];
1386
+ this._symbolCounts = [];
1387
+ this._symbolFacts = [];
605
1388
  this._symbolTableMetadata = [];
606
1389
  this._symbolIndexByValue = [];
607
1390
  if (this._df.columns.length === 0) {
@@ -614,33 +1397,144 @@ var init_QvdFileWriter = __esm({
614
1397
  const data = this._df.data;
615
1398
  const numColumns = columns.length;
616
1399
  const numRows = data.length;
1400
+ validateColumnNames(columns, this._path);
1401
+ const record = normaliseStoredSymbols(this._df.storedSymbols);
617
1402
  this._emitProgress("symbol-table", 0, numColumns);
618
- const indexByValue = columns.map(() => /* @__PURE__ */ new Map());
1403
+ const state = columns.map((name) => {
1404
+ const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
1405
+ const byKey = /* @__PURE__ */ new Map();
1406
+ let byValue = null;
1407
+ let textByNumber = null;
1408
+ if (entry !== null && valuesAreNumbers(entry)) {
1409
+ textByNumber = firstTextByValue(entry);
1410
+ } else if (entry !== null) {
1411
+ byValue = /* @__PURE__ */ new Map();
1412
+ for (let index = 0; index < entry.values.length; index++) {
1413
+ const found = byValue.get(entry.values[index]);
1414
+ if (found === void 0) {
1415
+ byValue.set(entry.values[index], index);
1416
+ } else if (typeof found === "number") {
1417
+ byValue.set(entry.values[index], [found, index]);
1418
+ } else {
1419
+ found.push(index);
1420
+ }
1421
+ }
1422
+ }
1423
+ return {
1424
+ name,
1425
+ keys: [],
1426
+ texts: [],
1427
+ byKey,
1428
+ // A cell is its own key without a record entry, and with one whose values are their numbers, so
1429
+ // one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
1430
+ byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
1431
+ byObject: /* @__PURE__ */ new Map(),
1432
+ entry,
1433
+ textByNumber,
1434
+ byValue,
1435
+ facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
1436
+ };
1437
+ });
1438
+ const byCells = state.map((column) => column.byCell);
1439
+ const byObjects = state.map((column) => column.byObject);
619
1440
  const containsNull = columns.map(() => false);
620
1441
  for (let row = 0; row < numRows; row++) {
621
1442
  const values = data[row];
1443
+ if (values !== null && values !== void 0 && !Array.isArray(values)) {
1444
+ throw new QvdValidationError("Each row must be an array of values", {
1445
+ row,
1446
+ type: typeof values,
1447
+ file: this._path,
1448
+ stage: "buildSymbolTable"
1449
+ });
1450
+ }
1451
+ if (values !== null && values !== void 0 && values.length > numColumns) {
1452
+ throw new QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
1453
+ row,
1454
+ values: values.length,
1455
+ fields: numColumns,
1456
+ file: this._path,
1457
+ stage: "buildSymbolTable"
1458
+ });
1459
+ }
622
1460
  for (let column = 0; column < numColumns; column++) {
623
1461
  const value = values?.[column];
624
1462
  if (value === null || value === void 0) {
625
1463
  containsNull[column] = true;
626
1464
  continue;
627
1465
  }
628
- const map = indexByValue[column];
629
- if (!map.has(value)) {
630
- map.set(value, map.size);
1466
+ if (typeof value === "object") {
1467
+ if (!byObjects[column].has(value)) {
1468
+ slotForObject(state[column], value, row, this._path);
1469
+ }
1470
+ continue;
1471
+ }
1472
+ if (byCells[column].has(value)) {
1473
+ continue;
1474
+ }
1475
+ if (typeof value === "number") {
1476
+ if (numberProblem(value) !== null) {
1477
+ checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
1478
+ }
1479
+ } else if (typeof value === "string") {
1480
+ if (textProblem(value) !== null) {
1481
+ checkText(value, "A string value", {
1482
+ column: columns[column],
1483
+ row,
1484
+ file: this._path,
1485
+ stage: "buildSymbolTable"
1486
+ });
1487
+ }
1488
+ } else {
1489
+ refuseCell(value, columns[column], row, this._path);
631
1490
  }
1491
+ byCells[column].set(value, slotForCell(state[column], value, row, this._path));
632
1492
  }
633
1493
  }
634
1494
  const columnBuffers = [];
635
1495
  let symbolsOffset = 0;
636
1496
  for (let column = 0; column < numColumns; column++) {
637
- const symbols = Array.from(indexByValue[column].keys(), (value) => _QvdFileWriter._convertRawToSymbol(value));
638
- const columnBuffer = Buffer.concat(symbols.map((symbol) => symbol.toByteRepresentation()));
1497
+ const { keys, texts } = state[column];
1498
+ const kinds = new Uint8Array(keys.length);
1499
+ let byteLength = 0;
1500
+ for (let slot = 0; slot < keys.length; slot++) {
1501
+ const key = keys[slot];
1502
+ const text = texts[slot];
1503
+ if (typeof key === "number") {
1504
+ if (numberProblem(key) !== null) {
1505
+ checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
1506
+ }
1507
+ if (text !== null && textProblem(text) !== null) {
1508
+ checkText(text, "The text of a dual value", {
1509
+ column: columns[column],
1510
+ file: this._path,
1511
+ stage: "buildSymbolTable"
1512
+ });
1513
+ }
1514
+ kinds[slot] = kindOf(key, text);
1515
+ byteLength += symbolByteLength(kinds[slot], key, text);
1516
+ } else {
1517
+ kinds[slot] = kindOf(null, key);
1518
+ byteLength += symbolByteLength(kinds[slot], null, key);
1519
+ }
1520
+ }
1521
+ const columnBuffer = Buffer.allocUnsafe(byteLength);
1522
+ let offset = 0;
1523
+ for (let slot = 0; slot < keys.length; slot++) {
1524
+ const key = keys[slot];
1525
+ offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
1526
+ }
1527
+ assert2(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
639
1528
  columnBuffers.push(columnBuffer);
640
- this._symbolTableMetadata?.push([symbolsOffset, columnBuffer.length, containsNull[column]]);
641
- this._symbolTable?.push(symbols);
642
- this._symbolIndexByValue.push(indexByValue[column]);
643
- symbolsOffset += columnBuffer.length;
1529
+ this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
1530
+ this._symbolCounts?.push(keys.length);
1531
+ this._symbolFacts?.push(state[column].facts);
1532
+ this._symbolIndexByValue?.push({
1533
+ byCell: state[column].byCell,
1534
+ byKey: state[column].byKey,
1535
+ byObject: state[column].byObject
1536
+ });
1537
+ symbolsOffset += byteLength;
644
1538
  this._emitProgress("symbol-table", column + 1, numColumns);
645
1539
  }
646
1540
  this._symbolBuffer = Buffer.concat(columnBuffers);
@@ -669,7 +1563,7 @@ var init_QvdFileWriter = __esm({
669
1563
  * @private
670
1564
  */
671
1565
  _buildIndexTable() {
672
- assert2(this._symbolTable, "The QVD file symbol table has not been built.");
1566
+ assert2(this._symbolCounts, "The QVD file symbol table has not been built.");
673
1567
  assert2(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
674
1568
  assert2(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
675
1569
  this._indexTableMetadata = [];
@@ -683,20 +1577,24 @@ var init_QvdFileWriter = __esm({
683
1577
  let totalBits = 0;
684
1578
  for (let column = 0; column < numColumns; column++) {
685
1579
  const fieldContainsNull = this._symbolTableMetadata[column][2];
686
- const symbolCount = this._symbolTable[column].length;
1580
+ const symbolCount = this._symbolCounts[column];
687
1581
  const nullShift = fieldContainsNull ? 2 : 0;
688
1582
  const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
689
1583
  const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
690
- for (const index of this._symbolIndexByValue[column].values()) {
691
- if (index + nullShift > maxStoredIndex) {
692
- throw new QvdValidationError("The symbol table and the index table are out of sync", {
693
- field: columns[column],
694
- storedIndex: index + nullShift,
695
- maxStoredIndex,
696
- symbolCount,
697
- file: this._path,
698
- stage: "buildIndexTable"
699
- });
1584
+ const lookup = this._symbolIndexByValue[column];
1585
+ const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
1586
+ for (const map of maps) {
1587
+ for (const index of map.values()) {
1588
+ if (index + nullShift > maxStoredIndex) {
1589
+ throw new QvdValidationError("The symbol table and the index table are out of sync", {
1590
+ field: columns[column],
1591
+ storedIndex: index + nullShift,
1592
+ maxStoredIndex,
1593
+ symbolCount,
1594
+ file: this._path,
1595
+ stage: "buildIndexTable"
1596
+ });
1597
+ }
700
1598
  }
701
1599
  }
702
1600
  layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
@@ -707,6 +1605,9 @@ var init_QvdFileWriter = __esm({
707
1605
  this._recordByteSize = recordByteSize;
708
1606
  this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
709
1607
  const progressInterval = Math.max(1, Math.floor(numRows / 100));
1608
+ const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
1609
+ const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
1610
+ const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
710
1611
  for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
711
1612
  const values = data[row];
712
1613
  for (let column = 0; column < numColumns; column++) {
@@ -714,7 +1615,7 @@ var init_QvdFileWriter = __esm({
714
1615
  if (value === null || value === void 0) {
715
1616
  continue;
716
1617
  }
717
- const index = this._symbolIndexByValue[column].get(value);
1618
+ const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
718
1619
  if (index === void 0) {
719
1620
  throw new QvdValidationError("A value is missing from the symbol table", {
720
1621
  field: columns[column],
@@ -731,29 +1632,6 @@ var init_QvdFileWriter = __esm({
731
1632
  }
732
1633
  this._symbolIndexByValue = null;
733
1634
  }
734
- /**
735
- * Converts a raw value/literal to a QVD symbol.
736
- *
737
- * @param {any} raw The raw value/literal to convert.
738
- * @return {QvdSymbol|null} The converted QVD symbol.
739
- */
740
- static _convertRawToSymbol(raw) {
741
- if (raw === null || raw === void 0) {
742
- return null;
743
- }
744
- const INT32_MIN = -2147483648;
745
- const INT32_MAX = 2147483647;
746
- const isInteger = typeof raw === "number" && Number.isInteger(raw);
747
- const isFloat = typeof raw === "number" && !Number.isInteger(raw);
748
- const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
749
- if (isInteger && isWithinInt32Range) {
750
- return QvdSymbol.fromDualIntValue(raw, raw.toString());
751
- } else if (isFloat || isInteger && !isWithinInt32Range) {
752
- return QvdSymbol.fromDualDoubleValue(raw, raw.toString());
753
- } else {
754
- return QvdSymbol.fromStringValue(raw);
755
- }
756
- }
757
1635
  /**
758
1636
  * Persists the data frame to a QVD file.
759
1637
  */
@@ -807,11 +1685,12 @@ function estimateRowMemory(rows, columnCount) {
807
1685
  }
808
1686
  return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
809
1687
  }
810
- function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
1688
+ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
811
1689
  const FULL_PARSE_OVERHEAD = 6;
812
1690
  const MINIMAL_OVERHEAD = 0.01;
813
1691
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
814
- const rowMemory = materialisesRows ? estimateRowMemory(rowsToLoad, columnCount) : BASE_BYTES;
1692
+ const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
1693
+ const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
815
1694
  if (maxRows === null || maxRows >= totalRows) {
816
1695
  return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
817
1696
  }
@@ -821,18 +1700,44 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
821
1700
  const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
822
1701
  return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
823
1702
  }
824
- function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true) {
825
- if (estimateMemoryUsage(symbolTableSize, totalRows, totalRows, columnCount, materialisesRows) <= budget) {
1703
+ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
1704
+ const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
1705
+ if (costOf(totalRows) <= budget) {
826
1706
  return totalRows;
827
1707
  }
828
- if (estimateMemoryUsage(symbolTableSize, 0, totalRows, columnCount, materialisesRows) > budget) {
1708
+ if (costOf(0) > budget) {
829
1709
  return 0;
830
1710
  }
831
1711
  let low = 0;
832
1712
  let high = totalRows;
833
1713
  while (high - low > 1) {
834
1714
  const mid = Math.floor((low + high) / 2);
835
- if (estimateMemoryUsage(symbolTableSize, mid, totalRows, columnCount, materialisesRows) <= budget) {
1715
+ if (costOf(mid) <= budget) {
1716
+ low = mid;
1717
+ } else {
1718
+ high = mid;
1719
+ }
1720
+ }
1721
+ return low;
1722
+ }
1723
+ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
1724
+ const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
1725
+ const fits = /* @__PURE__ */ __name((chunk) => {
1726
+ const live = Math.min(chunk * liveRowsPerChunk, covered);
1727
+ const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
1728
+ return cost <= budget;
1729
+ }, "fits");
1730
+ if (fits(covered)) {
1731
+ return covered;
1732
+ }
1733
+ if (!fits(1)) {
1734
+ return 0;
1735
+ }
1736
+ let low = 1;
1737
+ let high = covered;
1738
+ while (high - low > 1) {
1739
+ const mid = Math.floor((low + high) / 2);
1740
+ if (fits(mid)) {
836
1741
  low = mid;
837
1742
  } else {
838
1743
  high = mid;
@@ -840,7 +1745,7 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
840
1745
  }
841
1746
  return low;
842
1747
  }
843
- function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true) {
1748
+ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
844
1749
  if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
845
1750
  throw new QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
846
1751
  }
@@ -849,12 +1754,16 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
849
1754
  }
850
1755
  const budget = getMemoryBudget();
851
1756
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
852
- const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
853
- const externalMemory = estimateExternalMemory(rowsToLoad, columnCount);
1757
+ const rowsLive = live === null ? null : live.rows;
1758
+ const liveRowsPerChunk = live === null ? 1 : live.perChunk;
1759
+ const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
1760
+ const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
1761
+ const externalMemory = estimateExternalMemory(liveRows, columnCount);
854
1762
  const bounded = budget.candidates.map((candidate) => {
855
1763
  const heapOnly = candidate.source === "V8 heap limit";
856
1764
  return {
857
1765
  ...candidate,
1766
+ heapOnly,
858
1767
  needs: heapOnly ? heapMemory : heapMemory + externalMemory,
859
1768
  allowed: candidate.bytes * safetyFactor,
860
1769
  bounds: heapOnly ? "the V8 heap" : "the whole process"
@@ -870,12 +1779,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
870
1779
  const estimatedMemory = binding ? binding.needs : heapMemory;
871
1780
  const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
872
1781
  if (binding) {
1782
+ const includeExternal = !binding.heapOnly;
873
1783
  const recommendedMaxRows = recommendedRowsFor(
874
1784
  maxAllowedMemory,
875
1785
  symbolTableSize,
876
1786
  totalRows,
877
1787
  columnCount,
878
- materialisesRows
1788
+ materialisesRows,
1789
+ includeExternal
879
1790
  );
880
1791
  const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
881
1792
  const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
@@ -887,15 +1798,27 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
887
1798
  const limitingScope = binding.bounds;
888
1799
  const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
889
1800
  const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
890
- const nothingFits = recommendedMaxRows === 0;
891
1801
  const containerBound = binding.source === "container memory limit";
1802
+ const chunked = rowsLive !== null;
1803
+ const recommendedChunk = chunked ? recommendedChunkFor(
1804
+ maxAllowedMemory,
1805
+ symbolTableSize,
1806
+ maxRows,
1807
+ totalRows,
1808
+ columnCount,
1809
+ liveRowsPerChunk,
1810
+ includeExternal
1811
+ ) : 0;
1812
+ const knob = chunked ? "chunkSize" : "limit";
1813
+ const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
1814
+ const nothingFits = recommendedValue === 0;
892
1815
  let advice;
893
1816
  if (nothingFits) {
894
- advice = `No row count fits this budget - the symbol table alone exceeds it, so maxRows cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
1817
+ advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
895
1818
  } else if (containerBound) {
896
- advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`;
1819
+ advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
897
1820
  } else {
898
- advice = `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
1821
+ advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
899
1822
  }
900
1823
  throw new QvdValidationError(
901
1824
  `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
@@ -915,32 +1838,55 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
915
1838
  columnCount,
916
1839
  totalRows,
917
1840
  maxRows,
918
- recommendedMaxRows
1841
+ recommendedMaxRows,
1842
+ // Only present when a chunk size is what overflowed, so a caller cannot mistake one
1843
+ // recommendation for the other.
1844
+ ...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
919
1845
  }
920
1846
  );
921
1847
  }
922
1848
  }
1849
+ function formatCount(value) {
1850
+ return value.toLocaleString("en-US");
1851
+ }
923
1852
  function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
924
1853
  const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
925
- if (symbolTableSize > LARGE_SYMBOL_TABLE_WARNING && (maxRows === null || maxRows >= totalRows)) {
926
- const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
927
- const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
928
- const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
929
- const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
930
- console.warn(
931
- `\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
932
- );
1854
+ if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
1855
+ return;
1856
+ }
1857
+ const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
1858
+ const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
1859
+ if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
1860
+ return;
933
1861
  }
1862
+ const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
1863
+ const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
1864
+ const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
1865
+ console.warn(
1866
+ `\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
1867
+ );
934
1868
  }
935
1869
  var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
936
1870
  var init_memoryUtils = __esm({
937
1871
  "src/util/memoryUtils.js"() {
938
1872
  init_QvdErrors();
1873
+ __name(getHeapLimit, "getHeapLimit");
1874
+ __name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
1875
+ __name(getMemoryBudget, "getMemoryBudget");
939
1876
  HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
940
1877
  MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
1878
+ __name(usableOldSpaceLimit, "usableOldSpaceLimit");
941
1879
  BASE_BYTES = 16 * 1024 * 1024;
942
1880
  ROW_BASE_BYTES = 72;
943
1881
  PER_CELL_BYTES = 8;
1882
+ __name(estimateExternalMemory, "estimateExternalMemory");
1883
+ __name(estimateRowMemory, "estimateRowMemory");
1884
+ __name(estimateMemoryUsage, "estimateMemoryUsage");
1885
+ __name(recommendedRowsFor, "recommendedRowsFor");
1886
+ __name(recommendedChunkFor, "recommendedChunkFor");
1887
+ __name(validateMemoryAvailability, "validateMemoryAvailability");
1888
+ __name(formatCount, "formatCount");
1889
+ __name(warnLargeSymbolTable, "warnLargeSymbolTable");
944
1890
  }
945
1891
  });
946
1892
 
@@ -971,7 +1917,7 @@ function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
971
1917
  const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
972
1918
  const heapMB = Math.round(heapLimit / 1024 / 1024);
973
1919
  throw new QvdValidationError(
974
- `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
1920
+ `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
975
1921
  {
976
1922
  file: filePath,
977
1923
  symbolTableSize: symbolTableLength,
@@ -1045,7 +1991,7 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
1045
1991
  });
1046
1992
  }
1047
1993
  }
1048
- function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
1994
+ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
1049
1995
  if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
1050
1996
  throw new QvdCorruptedError("Invalid record byte size", {
1051
1997
  recordSize,
@@ -1107,23 +2053,28 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
1107
2053
  }
1108
2054
  }
1109
2055
  const requiredIndexBytes = rowsToLoad * recordSize;
1110
- if (indexTableOffset + requiredIndexBytes > bufferLength) {
2056
+ const bufferRecordStart = (windowFirstRow - bufferFirstRow) * recordSize;
2057
+ if (indexTableOffset + bufferRecordStart + requiredIndexBytes > bufferLength) {
1111
2058
  throw new QvdCorruptedError("Index table truncated", {
1112
2059
  indexTableOffset,
1113
2060
  requiredBytes: requiredIndexBytes,
1114
- availableBytes: Math.max(0, bufferLength - indexTableOffset),
2061
+ availableBytes: Math.max(0, bufferLength - indexTableOffset - bufferRecordStart),
1115
2062
  rowsToLoad,
2063
+ windowFirstRow,
2064
+ bufferFirstRow,
1116
2065
  recordSize,
1117
2066
  bufferSize: bufferLength,
1118
2067
  file: filePath,
1119
2068
  stage: "parseIndexTable"
1120
2069
  });
1121
2070
  }
1122
- if (indexTableLength < requiredIndexBytes) {
2071
+ const requiredTableBytes = (windowFirstRow + rowsToLoad) * recordSize;
2072
+ if (indexTableLength < requiredTableBytes) {
1123
2073
  throw new QvdCorruptedError("Index table length smaller than required", {
1124
2074
  indexTableLength,
1125
- requiredBytes: requiredIndexBytes,
2075
+ requiredBytes: requiredTableBytes,
1126
2076
  rowsToLoad,
2077
+ windowFirstRow,
1127
2078
  recordSize,
1128
2079
  file: filePath,
1129
2080
  stage: "parseIndexTable"
@@ -1184,262 +2135,235 @@ var init_validationUtils = __esm({
1184
2135
  init_QvdErrors();
1185
2136
  init_memoryUtils();
1186
2137
  init_bitUtils();
2138
+ __name(validateHeaderStructure, "validateHeaderStructure");
2139
+ __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
2140
+ __name(validateSymbolTableSize, "validateSymbolTableSize");
2141
+ __name(validateFieldMetadata, "validateFieldMetadata");
2142
+ __name(validateRecordCount, "validateRecordCount");
2143
+ __name(validateIndexTableMetadata, "validateIndexTableMetadata");
2144
+ __name(validateFieldBitMetadata, "validateFieldBitMetadata");
1187
2145
  }
1188
2146
  });
1189
2147
 
1190
2148
  // src/util/symbolParser.js
1191
- function parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1192
- if (pointer + 4 > bufferLength) {
1193
- throw new QvdCorruptedError("Buffer overflow reading integer symbol", {
1194
- field: fieldName,
1195
- pointer,
1196
- bufferSize: bufferLength,
1197
- file: filePath,
1198
- stage: "parseSymbolTable"
1199
- });
1200
- }
1201
- const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
1202
- const value = Buffer.from(byteData).readIntLE(0, byteData.length);
1203
- return { symbol: QvdSymbol.fromIntValue(value), bytesRead: 4 };
1204
- }
1205
- function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1206
- if (pointer + 8 > bufferLength) {
1207
- throw new QvdCorruptedError("Buffer overflow reading double symbol", {
1208
- field: fieldName,
1209
- pointer,
1210
- bufferSize: bufferLength,
1211
- file: filePath,
1212
- stage: "parseSymbolTable"
1213
- });
1214
- }
1215
- const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
1216
- const value = Buffer.from(byteData).readDoubleLE(0);
1217
- return { symbol: QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
1218
- }
1219
- function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1220
- const startPointer = pointer;
1221
- const maxStringLength = 1048576;
1222
- let stringLength = 0;
1223
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1224
- if (stringLength >= maxStringLength) {
1225
- throw new QvdCorruptedError("String symbol exceeds maximum length", {
1226
- field: fieldName,
1227
- maxLength: maxStringLength,
1228
- file: filePath,
1229
- stage: "parseSymbolTable"
1230
- });
1231
- }
1232
- pointer++;
1233
- stringLength++;
1234
- }
1235
- if (pointer >= bufferLength) {
1236
- throw new QvdCorruptedError("String symbol not null-terminated", {
2149
+ function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
2150
+ const bufferLength = symbolBuffer.length;
2151
+ const found = symbolBuffer.indexOf(0, from);
2152
+ if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
2153
+ throw new QvdCorruptedError(`${kind} exceeds maximum length`, {
1237
2154
  field: fieldName,
1238
- pointer,
1239
- bufferSize: bufferLength,
2155
+ maxLength: MAX_TEXT_BYTES,
1240
2156
  file: filePath,
1241
2157
  stage: "parseSymbolTable"
1242
2158
  });
1243
2159
  }
1244
- const value = symbolBuffer.subarray(startPointer, pointer).toString("utf-8");
1245
- return { symbol: QvdSymbol.fromStringValue(value), bytesRead: pointer - startPointer + 1 };
1246
- }
1247
- function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1248
- const startPointer = pointer;
1249
- const maxStringLength = 1048576;
1250
- let stringLength = 0;
1251
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1252
- if (stringLength >= maxStringLength) {
1253
- throw new QvdCorruptedError("String symbol exceeds maximum length", {
1254
- field: fieldName,
1255
- maxLength: maxStringLength,
1256
- file: filePath,
1257
- stage: "parseSymbolTable"
1258
- });
1259
- }
1260
- pointer++;
1261
- stringLength++;
1262
- }
1263
- if (pointer >= bufferLength) {
1264
- throw new QvdCorruptedError("String symbol not null-terminated", {
2160
+ if (found === -1) {
2161
+ throw new QvdCorruptedError(`${kind} not null-terminated`, {
1265
2162
  field: fieldName,
1266
- pointer,
2163
+ pointer: bufferLength,
1267
2164
  bufferSize: bufferLength,
1268
2165
  file: filePath,
1269
2166
  stage: "parseSymbolTable"
1270
2167
  });
1271
2168
  }
1272
- return pointer - startPointer + 1;
2169
+ return found;
1273
2170
  }
1274
- function parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1275
- if (pointer + 4 > bufferLength) {
1276
- throw new QvdCorruptedError("Buffer overflow reading dual integer symbol", {
1277
- field: fieldName,
1278
- pointer,
1279
- bufferSize: bufferLength,
1280
- file: filePath,
1281
- stage: "parseSymbolTable"
1282
- });
1283
- }
1284
- const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
1285
- const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
1286
- pointer += 4;
1287
- const stringStart = pointer;
1288
- const maxStringLength = 1048576;
1289
- let stringLength = 0;
1290
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1291
- if (stringLength >= maxStringLength) {
1292
- throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
1293
- field: fieldName,
1294
- maxLength: maxStringLength,
1295
- file: filePath,
1296
- stage: "parseSymbolTable"
1297
- });
1298
- }
1299
- pointer++;
1300
- stringLength++;
1301
- }
1302
- if (pointer >= bufferLength) {
1303
- throw new QvdCorruptedError("Dual string symbol not null-terminated", {
1304
- field: fieldName,
1305
- pointer,
1306
- bufferSize: bufferLength,
1307
- file: filePath,
1308
- stage: "parseSymbolTable"
1309
- });
1310
- }
1311
- const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
1312
- return { symbol: QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
2171
+ function overflow(message, pointer, bufferLength, fieldName, filePath) {
2172
+ throw new QvdCorruptedError(message, {
2173
+ field: fieldName,
2174
+ pointer,
2175
+ bufferSize: bufferLength,
2176
+ file: filePath,
2177
+ stage: "parseSymbolTable"
2178
+ });
1313
2179
  }
1314
- function parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1315
- if (pointer + 8 > bufferLength) {
1316
- throw new QvdCorruptedError("Buffer overflow reading dual double symbol", {
1317
- field: fieldName,
1318
- pointer,
1319
- bufferSize: bufferLength,
1320
- file: filePath,
1321
- stage: "parseSymbolTable"
1322
- });
1323
- }
1324
- const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
1325
- const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
1326
- pointer += 8;
1327
- const stringStart = pointer;
1328
- const maxStringLength = 1048576;
1329
- let stringLength = 0;
1330
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1331
- if (stringLength >= maxStringLength) {
1332
- throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
1333
- field: fieldName,
1334
- maxLength: maxStringLength,
1335
- file: filePath,
1336
- stage: "parseSymbolTable"
1337
- });
2180
+ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
2181
+ const bufferLength = symbolBuffer.length;
2182
+ const numbers = [];
2183
+ const texts = [];
2184
+ let pointer = start;
2185
+ while (pointer < end) {
2186
+ const typeByte = symbolBuffer[pointer++];
2187
+ const decode = keep === null || keep.has(numbers.length);
2188
+ let number = null;
2189
+ let text = null;
2190
+ switch (typeByte) {
2191
+ case 1: {
2192
+ if (decode) {
2193
+ if (pointer + 4 > bufferLength) {
2194
+ overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
2195
+ }
2196
+ number = symbolBuffer.readInt32LE(pointer);
2197
+ }
2198
+ pointer += 4;
2199
+ break;
2200
+ }
2201
+ case 2: {
2202
+ if (decode) {
2203
+ if (pointer + 8 > bufferLength) {
2204
+ overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
2205
+ }
2206
+ number = symbolBuffer.readDoubleLE(pointer);
2207
+ }
2208
+ pointer += 8;
2209
+ break;
2210
+ }
2211
+ case 4: {
2212
+ const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
2213
+ if (decode) {
2214
+ text = symbolBuffer.toString("utf8", pointer, terminator);
2215
+ }
2216
+ pointer = terminator + 1;
2217
+ break;
2218
+ }
2219
+ case 5:
2220
+ case 6: {
2221
+ const numberBytes = typeByte === 5 ? 4 : 8;
2222
+ if (pointer + numberBytes > bufferLength) {
2223
+ const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
2224
+ overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
2225
+ }
2226
+ const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
2227
+ if (decode) {
2228
+ number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
2229
+ text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
2230
+ }
2231
+ pointer = terminator + 1;
2232
+ break;
2233
+ }
2234
+ default: {
2235
+ throw new QvdParseError("Unknown symbol type byte", {
2236
+ typeByte: typeByte.toString(16),
2237
+ offset: pointer - 1,
2238
+ file: filePath,
2239
+ stage: "parseSymbolTable"
2240
+ });
2241
+ }
1338
2242
  }
1339
- pointer++;
1340
- stringLength++;
1341
- }
1342
- if (pointer >= bufferLength) {
1343
- throw new QvdCorruptedError("Dual string symbol not null-terminated", {
1344
- field: fieldName,
1345
- pointer,
1346
- bufferSize: bufferLength,
1347
- file: filePath,
1348
- stage: "parseSymbolTable"
1349
- });
2243
+ numbers.push(number);
2244
+ texts.push(text);
1350
2245
  }
1351
- const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
1352
- return {
1353
- symbol: QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
1354
- bytesRead: pointer - (stringStart - 8) + 1
1355
- };
2246
+ return { numbers, texts };
1356
2247
  }
1357
- function skipDualSymbol(symbolBuffer, pointer, bufferLength, numericBytes, fieldName, filePath) {
1358
- if (pointer + numericBytes > bufferLength) {
1359
- throw new QvdCorruptedError(`Buffer overflow reading dual symbol`, {
1360
- field: fieldName,
1361
- pointer,
1362
- bufferSize: bufferLength,
1363
- file: filePath,
1364
- stage: "parseSymbolTable"
1365
- });
2248
+ var MAX_TEXT_BYTES;
2249
+ var init_symbolParser = __esm({
2250
+ "src/util/symbolParser.js"() {
2251
+ init_QvdErrors();
2252
+ MAX_TEXT_BYTES = 1048576;
2253
+ __name(textEnd, "textEnd");
2254
+ __name(overflow, "overflow");
2255
+ __name(parseFieldSymbols, "parseFieldSymbols");
1366
2256
  }
1367
- const startPointer = pointer;
1368
- pointer += numericBytes;
1369
- const maxStringLength = 1048576;
1370
- let stringLength = 0;
1371
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1372
- if (stringLength >= maxStringLength) {
1373
- throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
1374
- field: fieldName,
1375
- maxLength: maxStringLength,
1376
- file: filePath,
1377
- stage: "parseSymbolTable"
1378
- });
2257
+ });
2258
+
2259
+ // src/util/resolveSymbols.js
2260
+ function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
2261
+ const length = symbols.numbers.length;
2262
+ const values = new Array(length);
2263
+ const entryValues = [];
2264
+ const numbers = [];
2265
+ const texts = [];
2266
+ let pure = 0;
2267
+ let partial = false;
2268
+ for (let index = 0; index < length; index++) {
2269
+ const text = symbols.texts[index];
2270
+ const number = symbols.numbers[index];
2271
+ if (text === null) {
2272
+ values[index] = number === null ? void 0 : number;
2273
+ if (number !== null) pure++;
2274
+ continue;
1379
2275
  }
1380
- pointer++;
1381
- stringLength++;
2276
+ if (number === null) {
2277
+ if (coerce && isNumericText(text)) {
2278
+ values[index] = Number(text);
2279
+ entryValues.push(values[index]);
2280
+ numbers.push(null);
2281
+ texts.push(text);
2282
+ partial = true;
2283
+ } else {
2284
+ values[index] = text;
2285
+ pure++;
2286
+ }
2287
+ continue;
2288
+ }
2289
+ partial = true;
2290
+ if (mode === "both") {
2291
+ values[index] = dualFromSymbol(number, text);
2292
+ continue;
2293
+ }
2294
+ values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
2295
+ entryValues.push(values[index]);
2296
+ numbers.push(number);
2297
+ texts.push(text);
1382
2298
  }
1383
- if (pointer >= bufferLength) {
1384
- throw new QvdCorruptedError("Dual string symbol not null-terminated", {
1385
- field: fieldName,
1386
- pointer,
1387
- bufferSize: bufferLength,
1388
- file: filePath,
1389
- stage: "parseSymbolTable"
2299
+ let entry = null;
2300
+ if (entryValues.length > 0) {
2301
+ entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
2302
+ field,
2303
+ values: Object.freeze(entryValues),
2304
+ numbers: Object.freeze(numbers),
2305
+ texts: Object.freeze(texts)
1390
2306
  });
1391
2307
  }
1392
- return pointer - startPointer + 1;
2308
+ return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
1393
2309
  }
1394
- function parseSymbol(typeByte, symbolBuffer, pointer, bufferLength, fieldName, filePath, shouldParse) {
1395
- switch (typeByte) {
1396
- case 1: {
1397
- if (shouldParse) {
1398
- return parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1399
- }
1400
- return { symbol: null, bytesRead: 4 };
2310
+ function collides(symbols, values, recorded) {
2311
+ for (let index = 0; index < values.length; index++) {
2312
+ if (!isPure(symbols, index, values[index])) {
2313
+ continue;
1401
2314
  }
1402
- case 2: {
1403
- if (shouldParse) {
1404
- return parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1405
- }
1406
- return { symbol: null, bytesRead: 8 };
1407
- }
1408
- case 4: {
1409
- if (shouldParse) {
1410
- return parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1411
- }
1412
- const bytesRead = skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1413
- return { symbol: null, bytesRead };
2315
+ if (recorded.has(values[index])) {
2316
+ return true;
1414
2317
  }
1415
- case 5: {
1416
- if (shouldParse) {
1417
- return parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1418
- }
1419
- const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 4, fieldName, filePath);
1420
- return { symbol: null, bytesRead };
2318
+ }
2319
+ return false;
2320
+ }
2321
+ function isPure(symbols, index, value) {
2322
+ const text = symbols.texts[index];
2323
+ const number = symbols.numbers[index];
2324
+ return text === null ? number !== null : number === null && value === text;
2325
+ }
2326
+ function collisionEntry(symbols, field, values, mode) {
2327
+ const recorded = /* @__PURE__ */ new Set();
2328
+ const inPassA = values.map((value, index) => {
2329
+ if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
2330
+ return false;
1421
2331
  }
1422
- case 6: {
1423
- if (shouldParse) {
1424
- return parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1425
- }
1426
- const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
1427
- return { symbol: null, bytesRead };
2332
+ const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
2333
+ if (recordedHere) {
2334
+ recorded.add(values[index]);
1428
2335
  }
1429
- default: {
1430
- throw new QvdParseError("Unknown symbol type byte", {
1431
- typeByte: typeByte.toString(16),
1432
- offset: pointer - 1,
1433
- file: filePath,
1434
- stage: "parseSymbolTable"
1435
- });
2336
+ return recordedHere;
2337
+ });
2338
+ const entryValues = [];
2339
+ const numbers = [];
2340
+ const texts = [];
2341
+ for (let index = 0; index < values.length; index++) {
2342
+ if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
2343
+ entryValues.push(values[index]);
2344
+ numbers.push(symbols.numbers[index]);
2345
+ texts.push(symbols.texts[index]);
1436
2346
  }
1437
2347
  }
2348
+ return Object.freeze({
2349
+ field,
2350
+ values: Object.freeze(entryValues),
2351
+ numbers: Object.freeze(numbers),
2352
+ texts: Object.freeze(texts)
2353
+ });
1438
2354
  }
1439
- var init_symbolParser = __esm({
1440
- "src/util/symbolParser.js"() {
1441
- init_QvdSymbol();
1442
- init_QvdErrors();
2355
+ function symbolHalves(symbols) {
2356
+ return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
2357
+ }
2358
+ var init_resolveSymbols = __esm({
2359
+ "src/util/resolveSymbols.js"() {
2360
+ init_QvdDual();
2361
+ init_cellRules();
2362
+ __name(resolveFieldSymbols, "resolveFieldSymbols");
2363
+ __name(collides, "collides");
2364
+ __name(isPure, "isPure");
2365
+ __name(collisionEntry, "collisionEntry");
2366
+ __name(symbolHalves, "symbolHalves");
1443
2367
  }
1444
2368
  });
1445
2369
 
@@ -1449,20 +2373,49 @@ __export(QvdColumnTable_exports, {
1449
2373
  QvdColumn: () => QvdColumn,
1450
2374
  QvdColumnTable: () => QvdColumnTable
1451
2375
  });
2376
+ function numberOf(value, number) {
2377
+ if (typeof value === "number") {
2378
+ return value;
2379
+ }
2380
+ if (typeof value === "string") {
2381
+ return number ?? NaN;
2382
+ }
2383
+ const dual = asDual(value);
2384
+ return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
2385
+ }
2386
+ function textOf(value) {
2387
+ if (typeof value === "string") {
2388
+ return value;
2389
+ }
2390
+ const dual = asDual(value);
2391
+ return dual !== null && typeof dual.text === "string" ? dual.text : null;
2392
+ }
1452
2393
  var QvdColumn, QvdColumnTable;
1453
2394
  var init_QvdColumnTable = __esm({
1454
2395
  "src/QvdColumnTable.js"() {
1455
2396
  init_QvdErrors();
2397
+ init_cellRules();
2398
+ init_readOptions();
2399
+ __name(numberOf, "numberOf");
2400
+ __name(textOf, "textOf");
1456
2401
  QvdColumn = class {
2402
+ static {
2403
+ __name(this, "QvdColumn");
2404
+ }
1457
2405
  /**
1458
2406
  * @param {string} name The field name.
1459
2407
  * @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
1460
2408
  * @param {Array<any>} symbols The field's distinct values, indexed by code.
2409
+ * @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
2410
+ * field whose values do not show them all - a dual read as one half, a string read as a number.
2411
+ * Without them, the halves are derived from the values: a string is its own text, a dual has its
2412
+ * own, and a number has none.
1461
2413
  */
1462
- constructor(name, codes, symbols) {
2414
+ constructor(name, codes, symbols, halves = null) {
1463
2415
  this._name = name;
1464
2416
  this._codes = codes;
1465
2417
  this._symbols = symbols;
2418
+ this._halves = halves;
1466
2419
  Object.freeze(this);
1467
2420
  }
1468
2421
  /** @return {string} The field name. */
@@ -1489,6 +2442,9 @@ var init_QvdColumnTable = __esm({
1489
2442
  *
1490
2443
  * One entry per distinct value, not per row: a few thousand entries for a column of millions.
1491
2444
  *
2445
+ * A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
2446
+ * entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
2447
+ *
1492
2448
  * @return {ReadonlyArray<any>} The dictionary.
1493
2449
  */
1494
2450
  get symbols() {
@@ -1512,6 +2468,42 @@ var init_QvdColumnTable = __esm({
1512
2468
  const code = this._codes[row];
1513
2469
  return code < 0 ? null : this._symbols[code];
1514
2470
  }
2471
+ /**
2472
+ * The text of one row: the text Qlik displays for its value.
2473
+ *
2474
+ * A string is its own text and a dual has its own. A value read as one half of a symbol - a date
2475
+ * read as its serial, a string read as a number - has the text the file stores for it, and a pure
2476
+ * number has none.
2477
+ *
2478
+ * @param {number} row The row index.
2479
+ * @return {string|null} The text, or null for NULL and for a number with no text.
2480
+ * @throws {QvdValidationError} If the row is not an integer within the column.
2481
+ */
2482
+ textAt(row) {
2483
+ if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
2484
+ throw new QvdValidationError("Row index out of bounds", {
2485
+ column: this._name,
2486
+ row,
2487
+ length: this._codes.length
2488
+ });
2489
+ }
2490
+ const code = this._codes[row];
2491
+ if (code < 0) {
2492
+ return null;
2493
+ }
2494
+ return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
2495
+ }
2496
+ /**
2497
+ * The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
2498
+ *
2499
+ * @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
2500
+ */
2501
+ symbolTexts() {
2502
+ if (this._halves !== null) {
2503
+ return this._halves.texts;
2504
+ }
2505
+ return Object.freeze(this._symbols.map(textOf));
2506
+ }
1515
2507
  /**
1516
2508
  * Iterates the column's values without materialising it.
1517
2509
  *
@@ -1555,6 +2547,9 @@ var init_QvdColumnTable = __esm({
1555
2547
  * one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
1556
2548
  * codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
1557
2549
  *
2550
+ * A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
2551
+ * `{duals: 'text'}` gives the serial the file stores for it, not NaN.
2552
+ *
1558
2553
  * Scanning `codes` against this is the fastest way to read a column, because both sides are
1559
2554
  * contiguous typed arrays and the dictionary fits in cache:
1560
2555
  *
@@ -1576,8 +2571,7 @@ var init_QvdColumnTable = __esm({
1576
2571
  numericSymbols() {
1577
2572
  const out = new Float64Array(this._symbols.length);
1578
2573
  for (let index = 0; index < this._symbols.length; index++) {
1579
- const value = this._symbols[index];
1580
- out[index] = typeof value === "number" ? value : NaN;
2574
+ out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
1581
2575
  }
1582
2576
  return out;
1583
2577
  }
@@ -1592,7 +2586,8 @@ var init_QvdColumnTable = __esm({
1592
2586
  * @param {Object} [options] Conversion options.
1593
2587
  * @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
1594
2588
  * number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
1595
- * NaN, which is the right choice only when the caller knows the column is numeric.
2589
+ * NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
2590
+ * number here, as it is to `numericSymbols`.
1596
2591
  * @return {Float64Array} One number per row.
1597
2592
  * @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
1598
2593
  */
@@ -1612,6 +2607,11 @@ var init_QvdColumnTable = __esm({
1612
2607
  out[row] = value;
1613
2608
  continue;
1614
2609
  }
2610
+ const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
2611
+ if (!Number.isNaN(number)) {
2612
+ out[row] = number;
2613
+ continue;
2614
+ }
1615
2615
  if (onNonNumeric === "throw") {
1616
2616
  throw new QvdValidationError("Column holds a value that is not a number", {
1617
2617
  column: this._name,
@@ -1627,29 +2627,57 @@ var init_QvdColumnTable = __esm({
1627
2627
  }
1628
2628
  };
1629
2629
  QvdColumnTable = class {
2630
+ static {
2631
+ __name(this, "QvdColumnTable");
2632
+ }
1630
2633
  /**
1631
2634
  * @param {Object} decoded What the reader decoded.
1632
2635
  * @param {Array<string>} decoded.columns Field names, in file order.
1633
2636
  * @param {Array<Int32Array>} decoded.codesByField One code array per field.
1634
2637
  * @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
2638
+ * @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
2639
+ * or null for a field whose values show them.
1635
2640
  * @param {number} decoded.rowCount Rows decoded.
1636
2641
  * @param {any} decoded.metadata The raw QvdTableHeader.
2642
+ * @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
2643
+ * stored-symbol record, as a data frame of the same read carries it.
1637
2644
  * @param {any} decoded.loadStats Statistics about the read.
1638
2645
  */
1639
- constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
2646
+ constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
1640
2647
  this._columns = columns;
1641
2648
  this._codesByField = codesByField;
1642
2649
  this._symbolsByField = symbolsByField;
2650
+ this._halvesByField = halvesByField ?? null;
1643
2651
  this._rowCount = rowCount;
1644
2652
  this._metadata = metadata;
2653
+ this._storedSymbols = storedSymbols ?? null;
1645
2654
  this._loadStats = loadStats;
1646
2655
  }
1647
2656
  /**
1648
2657
  * Reads a QVD file as columns.
1649
2658
  *
2659
+ * Takes the same options as `QvdDataFrame.fromQvd`, with the same meanings - one option
2660
+ * vocabulary for both read paths, because they are two answers about the same file rather than
2661
+ * two features. `{offset, limit}` is how a caller pages through a file columnwise; there is no
2662
+ * columnar `iterate()` because there is nothing for it to bound - a columnar read materialises
2663
+ * no rows, which is the memory chunking exists to cap.
2664
+ *
1650
2665
  * @param {string} path The path to the QVD file.
1651
2666
  * @param {Object} [options] Loading options, with the same meanings they have on `fromQvd`.
1652
- * @param {number|null} [options.maxRows] Maximum rows to decode.
2667
+ * @param {number|null} [options.maxRows] Rows to decode. The older name for `limit`.
2668
+ * @param {number|null} [options.limit] Rows to decode, counting from `offset`.
2669
+ * @param {number} [options.offset] File row to start at.
2670
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
2671
+ * appear. Unselected fields have their symbols skipped entirely.
2672
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
2673
+ * number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
2674
+ * text and `numericSymbols` the number. Anything else throws.
2675
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
2676
+ * number when its text is not blank and `Number(text)` is finite - a string symbol as
2677
+ * `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
2678
+ * Anything but a boolean throws.
2679
+ * @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
2680
+ * @param {AbortSignal} [options.signal] Cancels the read.
1653
2681
  * @param {string} [options.allowedDir] Directory the path must resolve inside.
1654
2682
  * @param {number} [options.memorySafetyFactor] Fraction of the memory budget a load may use.
1655
2683
  * @param {number} [options.symbolFilteringThreshold] Symbol table size above which a limited
@@ -1659,15 +2687,13 @@ var init_QvdColumnTable = __esm({
1659
2687
  static async fromQvd(path3, options = {}) {
1660
2688
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
1661
2689
  const reader = new QvdFileReader2(path3, {
1662
- allowedDir: options.allowedDir,
1663
- memorySafetyFactor: options.memorySafetyFactor,
1664
- symbolFilteringThreshold: options.symbolFilteringThreshold,
2690
+ ...readerOptionsFrom(options),
1665
2691
  // This read builds no rows, so the memory guard must not charge it for them. A columnar
1666
2692
  // read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
1667
2693
  // refused below a 2GB one.
1668
2694
  materialisesRows: false
1669
2695
  });
1670
- return await reader.loadColumnar(options.maxRows !== void 0 ? options.maxRows : null);
2696
+ return await reader.loadColumnar(windowFrom(options));
1671
2697
  }
1672
2698
  /** @return {Array<string>} Field names, in file order. */
1673
2699
  get columns() {
@@ -1689,6 +2715,14 @@ var init_QvdColumnTable = __esm({
1689
2715
  get loadStats() {
1690
2716
  return this._loadStats;
1691
2717
  }
2718
+ /**
2719
+ * The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
2720
+ *
2721
+ * @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
2722
+ */
2723
+ get storedSymbols() {
2724
+ return this._storedSymbols;
2725
+ }
1692
2726
  /**
1693
2727
  * One column.
1694
2728
  *
@@ -1704,7 +2738,12 @@ var init_QvdColumnTable = __esm({
1704
2738
  availableColumns: this._columns
1705
2739
  });
1706
2740
  }
1707
- return new QvdColumn(name, this._codesByField[index], this._symbolsByField[index]);
2741
+ return new QvdColumn(
2742
+ name,
2743
+ this._codesByField[index],
2744
+ this._symbolsByField[index],
2745
+ this._halvesByField?.[index] ?? null
2746
+ );
1708
2747
  }
1709
2748
  };
1710
2749
  }
@@ -1715,7 +2754,17 @@ var QvdFileReader_exports = {};
1715
2754
  __export(QvdFileReader_exports, {
1716
2755
  QvdFileReader: () => QvdFileReader
1717
2756
  });
1718
- var MAX_HEADER_SIZE, READ_CHUNK_SIZE, QvdFileReader;
2757
+ function closeReadStream(stream) {
2758
+ if (stream.closed) {
2759
+ return Promise.resolve();
2760
+ }
2761
+ return new Promise((resolve) => {
2762
+ stream.once("close", () => resolve());
2763
+ stream.once("error", () => resolve());
2764
+ stream.destroy();
2765
+ });
2766
+ }
2767
+ var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
1719
2768
  var init_QvdFileReader = __esm({
1720
2769
  "src/QvdFileReader.js"() {
1721
2770
  init_QvdDataFrame();
@@ -1725,9 +2774,17 @@ var init_QvdFileReader = __esm({
1725
2774
  init_memoryUtils();
1726
2775
  init_validationUtils();
1727
2776
  init_symbolParser();
2777
+ init_readOptions();
2778
+ init_resolveSymbols();
2779
+ init_storedSymbols();
1728
2780
  MAX_HEADER_SIZE = 16 * 1024 * 1024;
1729
2781
  READ_CHUNK_SIZE = 512 * 1024 * 1024;
2782
+ ANALYSIS_SLICE_ROWS = 65536;
2783
+ __name(closeReadStream, "closeReadStream");
1730
2784
  QvdFileReader = class {
2785
+ static {
2786
+ __name(this, "QvdFileReader");
2787
+ }
1731
2788
  /**
1732
2789
  * Constructs a new QVD file parser.
1733
2790
  *
@@ -1738,9 +2795,10 @@ var init_QvdFileReader = __esm({
1738
2795
  * points outside it is rejected. Defaults to the current working directory. To permit
1739
2796
  * an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
1740
2797
  * empty value falls back to the working directory rather than removing the restriction.
1741
- * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a
1742
- * load may use. The budget is the smallest of the V8 heap limit, any container memory limit,
1743
- * and the memory the OS reports as available. Default is 0.3. **Zero disables the memory
2798
+ * @param {number} [options.memorySafetyFactor=0.8] Fraction (0.0-1.0) of the memory budget a
2799
+ * load may use. The budget is the smaller of the V8 heap limit and any container memory limit;
2800
+ * what the OS reports as available is recorded for diagnostics and deliberately not allowed to
2801
+ * bind - see `getMemoryBudget`. Default is 0.8. **Zero disables the memory
1744
2802
  * check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
1745
2803
  * Bun reports its current heap as its heap limit - and for callers who would rather manage
1746
2804
  * memory themselves than trust the estimate.
@@ -1751,41 +2809,123 @@ var init_QvdFileReader = __esm({
1751
2809
  * above which a lazy load switches to the two-pass filtering path. The default of 50MB is
1752
2810
  * the point where the extra analysis pass pays for itself; lower it to use filtering on
1753
2811
  * smaller files, raise it to keep the simpler single-pass read for longer.
2812
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
2813
+ * appear. Null reads every field, in file order. An unknown or repeated name is refused.
2814
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
2815
+ * text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
2816
+ * value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
2817
+ * `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
2818
+ * `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
2819
+ * the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
2820
+ * throws a `QvdValidationError`.
2821
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
2822
+ * reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
2823
+ * every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
2824
+ * dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
2825
+ * dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
2826
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
2827
+ * read proceeds - the same shape `QvdFileWriter` emits.
2828
+ * @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
2829
+ * `signal.reason`, exactly as `signal.throwIfAborted()` does.
1754
2830
  */
1755
2831
  constructor(filePath, options = {}) {
1756
2832
  const {
1757
2833
  allowedDir,
1758
2834
  memorySafetyFactor = 0.8,
1759
2835
  symbolFilteringThreshold = 50 * 1024 * 1024,
1760
- materialisesRows = true
2836
+ materialisesRows = true,
2837
+ fields = null,
2838
+ duals,
2839
+ coerceNumericStrings,
2840
+ onProgress,
2841
+ signal
1761
2842
  } = options;
1762
2843
  this._materialisesRows = materialisesRows;
1763
2844
  this._path = validatePath(filePath, allowedDir);
2845
+ this._duals = normaliseDuals(duals, this._path);
2846
+ this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
1764
2847
  this._memorySafetyFactor = memorySafetyFactor;
1765
2848
  this._symbolFilteringThreshold = symbolFilteringThreshold;
2849
+ if (onProgress !== void 0 && typeof onProgress !== "function") {
2850
+ throw new QvdValidationError("onProgress must be a function", {
2851
+ provided: onProgress,
2852
+ type: typeof onProgress,
2853
+ file: this._path
2854
+ });
2855
+ }
2856
+ if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
2857
+ throw new QvdValidationError("signal must be an AbortSignal", {
2858
+ provided: signal,
2859
+ type: typeof signal,
2860
+ file: this._path
2861
+ });
2862
+ }
2863
+ this._requestedFields = fields === void 0 ? null : fields;
2864
+ this._onProgress = onProgress;
2865
+ this._signal = signal;
1766
2866
  this._buffer = null;
1767
2867
  this._headerOffset = null;
1768
2868
  this._symbolTableOffset = null;
1769
2869
  this._indexTableOffset = null;
1770
2870
  this._header = null;
2871
+ this._allFields = null;
2872
+ this._selectedFields = null;
2873
+ this._fieldBitMetadataValidated = false;
1771
2874
  this._symbolTable = null;
1772
2875
  this._indexColumns = null;
1773
2876
  this._rowsDecoded = 0;
2877
+ this._bufferFirstRow = 0;
1774
2878
  this._fileSize = null;
1775
2879
  this._headerMatchesFile = false;
1776
2880
  }
2881
+ /**
2882
+ * Emits a progress event if a callback is registered.
2883
+ *
2884
+ * The same shape `QvdFileWriter._emitProgress` emits, deliberately: a caller who has written a
2885
+ * progress bar for a write should not have to write a second one for a read. The stage names
2886
+ * differ because the stages differ, but `symbol-table` and `index-table` mean the same thing on
2887
+ * both sides.
2888
+ *
2889
+ * @param {string} stage The current stage of the read.
2890
+ * @param {number} current The current progress value.
2891
+ * @param {number} total The total progress value.
2892
+ * @private
2893
+ */
2894
+ _emitProgress(stage, current, total) {
2895
+ if (this._onProgress) {
2896
+ const percent = total > 0 ? Math.round(current / total * 100) : 100;
2897
+ this._onProgress({ stage, current, total, percent });
2898
+ }
2899
+ }
2900
+ /**
2901
+ * Throws if the caller has cancelled the read.
2902
+ *
2903
+ * Throws `signal.reason` - a `DOMException` named `AbortError` unless the caller aborted with a
2904
+ * reason of their own. That is what `AbortSignal` means everywhere else in Node, and inventing
2905
+ * a `QvdAbortError` here would make this library's cancellation the one a caller has to special
2906
+ * case.
2907
+ *
2908
+ * @private
2909
+ */
2910
+ _throwIfAborted() {
2911
+ if (this._signal) {
2912
+ this._signal.throwIfAborted();
2913
+ }
2914
+ }
1777
2915
  /**
1778
2916
  * Reads the binary data of the QVD file.
1779
2917
  *
1780
- * LAZY LOADING OPTIMIZATION: When maxRows is specified, this method implements
1781
- * true lazy loading by reading only the necessary portions of the file from disk.
2918
+ * A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
2919
+ * needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
2920
+ * over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
1782
2921
  *
1783
- * For large files (e.g., 5GB), loading only the first 1000 rows can save significant
1784
- * memory and time:
1785
- * - Full load: 5GB in memory, ~30-60s load time
1786
- * - Lazy load (maxRows=1000): ~1.75-2GB in memory, ~2-5s load time
2922
+ * The saving is in the index table and the rows, not in the symbol table, which is read in
2923
+ * full whatever the window because a stored index in any row can address any symbol. So the
2924
+ * gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
2925
+ * values there is very little to save, which is what `symbolFilteringThreshold` and the
2926
+ * two-pass path exist for.
1787
2927
  *
1788
- * Algorithm for Lazy Loading:
2928
+ * Algorithm for a windowed read:
1789
2929
  * 1. Stream-read the file until XML header delimiter is found
1790
2930
  * 2. Parse header to determine symbol table and index table locations
1791
2931
  * 3. Calculate bytes needed: header + full symbol table + partial index table
@@ -1798,14 +2938,23 @@ var init_QvdFileReader = __esm({
1798
2938
  * - Streaming for header finding is efficient for unknown header sizes
1799
2939
  * - Direct byte-range reading for remaining data is fastest
1800
2940
  *
1801
- * @param {number|null} maxRows The maximum number of rows to load. If null, all data is loaded.
2941
+ * A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
2942
+ * table from the front of the file, and the window's records from wherever they sit. The bytes
2943
+ * between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
2944
+ * fixture a 0.4MB read rather than a 38MB one.
2945
+ *
2946
+ * @param {QvdRowWindow} window The rows to read.
1802
2947
  * @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
1803
2948
  * symbol and index tables on disk. This is the metadata-only path: the header is a few
1804
2949
  * kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
1805
2950
  * for a 40GB one.
2951
+ * @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
2952
+ * that is fewer than the window covers - see `_prepare`.
1806
2953
  * @private
1807
2954
  */
1808
- async _readData(maxRows = null, headerOnly = false) {
2955
+ async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
2956
+ this._throwIfAborted();
2957
+ this._emitProgress("read", 0, 1);
1809
2958
  const HEADER_DELIMITER = "\r\n\0";
1810
2959
  const CHUNK_SIZE = 64 * 1024;
1811
2960
  const stream = fs.createReadStream(this._path, {
@@ -1847,6 +2996,8 @@ var init_QvdFileReader = __esm({
1847
2996
  if (!isExpectedEarlyClose) {
1848
2997
  throw error;
1849
2998
  }
2999
+ } finally {
3000
+ await closeReadStream(stream);
1850
3001
  }
1851
3002
  if (headerDelimiterIndex === -1) {
1852
3003
  throw new QvdCorruptedError(
@@ -1875,13 +3026,14 @@ var init_QvdFileReader = __esm({
1875
3026
  const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
1876
3027
  if (headerOnly) {
1877
3028
  this._buffer = headerBuffer.subarray(0, headerEndIndex);
3029
+ this._emitProgress("read", 1, 1);
1878
3030
  return;
1879
3031
  }
1880
3032
  let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
1881
3033
  if (headerFields && !Array.isArray(headerFields)) {
1882
3034
  headerFields = [headerFields];
1883
3035
  }
1884
- const columnCount = Array.isArray(headerFields) ? headerFields.length : 0;
3036
+ const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
1885
3037
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
1886
3038
  (value) => Number.isSafeInteger(value) && value >= 0
1887
3039
  );
@@ -1890,23 +3042,28 @@ var init_QvdFileReader = __esm({
1890
3042
  this._fileSize = fileSize;
1891
3043
  this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
1892
3044
  }
3045
+ const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
3046
+ const windowRows = resolved.limit;
1893
3047
  if (headerNumbersUsable && this._headerMatchesFile) {
1894
3048
  validateMemoryAvailability(
1895
3049
  symbolTableLength,
1896
- maxRows,
3050
+ windowRows,
1897
3051
  totalRows,
1898
3052
  this._path,
1899
3053
  this._memorySafetyFactor,
1900
3054
  columnCount,
1901
- this._materialisesRows
3055
+ this._materialisesRows,
3056
+ liveRows
1902
3057
  );
1903
3058
  }
1904
- if (maxRows === null) {
3059
+ if (window.offset === 0 && window.limit === null) {
1905
3060
  this._buffer = await fs.promises.readFile(this._path);
1906
3061
  this._fileSize = this._buffer.length;
3062
+ this._bufferFirstRow = 0;
3063
+ this._emitProgress("read", 1, 1);
1907
3064
  return;
1908
3065
  }
1909
- const rowsToLoad = Math.min(maxRows, totalRows);
3066
+ const rowsToLoad = windowRows;
1910
3067
  validateSymbolTableSizeEarly(symbolTableLength, this._path);
1911
3068
  for (const [name, value] of [
1912
3069
  ["Offset", symbolTableLength],
@@ -1922,39 +3079,78 @@ var init_QvdFileReader = __esm({
1922
3079
  });
1923
3080
  }
1924
3081
  }
3082
+ const skippedIndexBytes = resolved.offset * recordSize;
1925
3083
  const indexTableBytesToRead = rowsToLoad * recordSize;
1926
3084
  const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
3085
+ const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
1927
3086
  const fd = await fs.promises.open(this._path, "r");
1928
3087
  try {
1929
3088
  const { size: fileSize } = await fd.stat();
1930
3089
  this._fileSize = fileSize;
1931
- if (totalBytesToRead > fileSize) {
3090
+ if (fileBytesRequired > fileSize) {
1932
3091
  throw new QvdCorruptedError("The file is shorter than its header claims.", {
1933
3092
  file: this._path,
1934
3093
  fileSize,
1935
- requiredBytes: totalBytesToRead,
3094
+ requiredBytes: fileBytesRequired,
1936
3095
  stage: "readData"
1937
3096
  });
1938
3097
  }
1939
3098
  this._buffer = Buffer.alloc(totalBytesToRead);
1940
- let position = 0;
1941
- while (position < totalBytesToRead) {
1942
- const length = Math.min(READ_CHUNK_SIZE, totalBytesToRead - position);
1943
- const { bytesRead } = await fd.read(this._buffer, position, length, position);
1944
- if (bytesRead === 0) {
1945
- throw new QvdCorruptedError("Unexpected end of file while reading QVD data.", {
1946
- file: this._path,
1947
- fileSize,
1948
- bytesRead: position,
1949
- requiredBytes: totalBytesToRead,
1950
- stage: "readData"
1951
- });
1952
- }
1953
- position += bytesRead;
3099
+ await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
3100
+ if (indexTableBytesToRead > 0) {
3101
+ await this._readRange(
3102
+ fd,
3103
+ indexTableOffset,
3104
+ indexTableBytesToRead,
3105
+ indexTableOffset + skippedIndexBytes,
3106
+ fileSize,
3107
+ fileBytesRequired
3108
+ );
1954
3109
  }
3110
+ this._bufferFirstRow = resolved.offset;
1955
3111
  } finally {
1956
3112
  await fd.close();
1957
3113
  }
3114
+ this._emitProgress("read", 1, 1);
3115
+ }
3116
+ /**
3117
+ * Reads one byte range of the file into the buffer.
3118
+ *
3119
+ * Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
3120
+ * 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
3121
+ * try/catch can intercept.
3122
+ *
3123
+ * @param {import('fs/promises').FileHandle} fd The open file.
3124
+ * @param {number} bufferOffset Where in the buffer to write.
3125
+ * @param {number} byteCount How many bytes to read.
3126
+ * @param {number} filePosition Where in the file to read from.
3127
+ * @param {number} fileSize The file's size, for the error.
3128
+ * @param {number} requiredBytes Bytes the whole read needs, for the error.
3129
+ * @private
3130
+ */
3131
+ async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
3132
+ assert2(this._buffer, "The read buffer has not been allocated.");
3133
+ let done = 0;
3134
+ while (done < byteCount) {
3135
+ const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
3136
+ const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
3137
+ if (bytesRead === 0) {
3138
+ throw new QvdCorruptedError("Unexpected end of file while reading QVD data.", {
3139
+ file: this._path,
3140
+ fileSize,
3141
+ // Two numbers, because they stopped being the same one when a window began reading two
3142
+ // ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
3143
+ // file it gave up. Reporting the position under the name of the count made a windowed
3144
+ // read of a truncated file claim tens of megabytes had been read when a few hundred
3145
+ // bytes had.
3146
+ bytesRead: done,
3147
+ filePosition: filePosition + done,
3148
+ requiredBytes,
3149
+ stage: "readData"
3150
+ });
3151
+ }
3152
+ done += bytesRead;
3153
+ }
1958
3154
  }
1959
3155
  /**
1960
3156
  * Parses the XML header of the QVD file. This method is part of the parsing process
@@ -2014,6 +3210,8 @@ var init_QvdFileReader = __esm({
2014
3210
  this._headerOffset = headerBeginIndex;
2015
3211
  this._symbolTableOffset = headerEndIndex;
2016
3212
  this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3213
+ this._allFields = fieldList;
3214
+ this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
2017
3215
  }
2018
3216
  /**
2019
3217
  * Establishes the geometry of the index table, and validates it.
@@ -2024,30 +3222,29 @@ var init_QvdFileReader = __esm({
2024
3222
  * about keeping the sign in step with the other one: the two could drift, and #113 is what
2025
3223
  * that looks like when they do. There is one copy now.
2026
3224
  *
2027
- * @param {number|null} rowLimit Maximum rows of interest, or null for all of them.
3225
+ * @param {QvdRowWindow} window The rows of interest, as file row indices.
2028
3226
  * @param {string} stage Stage name for any error raised here.
2029
3227
  * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
2030
- * indexBuffer: Buffer}} The record geometry.
3228
+ * indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
3229
+ * record, so the decoder always counts from zero.
2031
3230
  * @private
2032
3231
  */
2033
- _planIndexTable(rowLimit, stage) {
2034
- if (!this._buffer || !this._header || !this._indexTableOffset) {
3232
+ _planIndexTable(window, stage) {
3233
+ if (!this._buffer || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
2035
3234
  throw new QvdCorruptedError(
2036
3235
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
2037
3236
  {
2038
3237
  file: this._path,
2039
3238
  stage
2040
3239
  }
2041
- );
2042
- }
2043
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
2044
- if (!Array.isArray(fields)) {
2045
- fields = [fields];
3240
+ );
2046
3241
  }
3242
+ const allFields = this._allFields;
3243
+ const fields = this._selectedFields;
2047
3244
  const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
2048
3245
  const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
2049
- const rowsToLoad = rowLimit !== null ? Math.min(rowLimit, totalRows) : totalRows;
2050
3246
  const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
3247
+ const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
2051
3248
  validateIndexTableMetadata(
2052
3249
  recordSize,
2053
3250
  totalRows,
@@ -2056,11 +3253,20 @@ var init_QvdFileReader = __esm({
2056
3253
  this._buffer.length,
2057
3254
  rowsToLoad,
2058
3255
  this._path,
2059
- this._fileSize
3256
+ this._fileSize,
3257
+ firstRow,
3258
+ this._bufferFirstRow
3259
+ );
3260
+ const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
3261
+ const indexBuffer = this._buffer.subarray(
3262
+ this._indexTableOffset + bufferRecordStart,
3263
+ this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
2060
3264
  );
2061
- const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
2062
- for (const field of fields) {
2063
- validateFieldBitMetadata(field, recordSize, this._path);
3265
+ if (!this._fieldBitMetadataValidated) {
3266
+ for (const field of allFields) {
3267
+ validateFieldBitMetadata(field, recordSize, this._path);
3268
+ }
3269
+ this._fieldBitMetadataValidated = true;
2064
3270
  }
2065
3271
  assert2(
2066
3272
  rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
@@ -2072,31 +3278,45 @@ var init_QvdFileReader = __esm({
2072
3278
  * Analyzes the index table to determine which symbols are actually needed.
2073
3279
  * This is used for two-pass symbol filtering optimization.
2074
3280
  *
2075
- * @param {number} maxRows The maximum number of rows to analyze.
2076
- * @return {Promise<Map<string, Set<number>>>} Map of field names to Set of needed symbol indices.
3281
+ * Only the selected fields are analysed. An unselected field's symbols are never parsed, so
3282
+ * there is nothing for a usage set to filter and decoding its column would be a pass over the
3283
+ * whole window for an answer nobody reads.
3284
+ *
3285
+ * @param {QvdRowWindow} window The rows to analyse.
3286
+ * @return {Promise<Array<Set<number>>>} One set of needed symbol indices per selected field, in
3287
+ * the same order `_parseSymbolTable` walks them.
2077
3288
  * @private
2078
3289
  */
2079
- async _analyzeIndexTableSymbolUsage(maxRows) {
2080
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "analyzeIndexTableSymbolUsage");
2081
- const symbolUsage = /* @__PURE__ */ new Map();
2082
- const column = new Int32Array(rowsToLoad);
2083
- fields.forEach((field) => {
3290
+ async _analyzeIndexTableSymbolUsage(window) {
3291
+ const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
3292
+ const symbolUsage = [];
3293
+ const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
3294
+ const column = new Int32Array(sliceRows);
3295
+ fields.forEach((field, position) => {
3296
+ this._throwIfAborted();
2084
3297
  const needed = /* @__PURE__ */ new Set();
2085
- symbolUsage.set(field["FieldName"], needed);
2086
- decodeIndexColumn(
2087
- indexBuffer,
2088
- recordSize,
2089
- rowsToLoad,
2090
- parseInt(field["BitOffset"], 10),
2091
- parseInt(field["BitWidth"], 10),
2092
- parseInt(field["Bias"], 10),
2093
- column
2094
- );
2095
- for (let row = 0; row < rowsToLoad; row++) {
2096
- if (column[row] >= 0) {
2097
- needed.add(column[row]);
3298
+ symbolUsage[position] = needed;
3299
+ const bitOffset = parseInt(field["BitOffset"], 10);
3300
+ const bitWidth = parseInt(field["BitWidth"], 10);
3301
+ const bias = parseInt(field["Bias"], 10);
3302
+ for (let first = 0; first < rowsToLoad; first += sliceRows) {
3303
+ const count = Math.min(sliceRows, rowsToLoad - first);
3304
+ decodeIndexColumn(
3305
+ first === 0 ? indexBuffer : indexBuffer.subarray(first * recordSize),
3306
+ recordSize,
3307
+ count,
3308
+ bitOffset,
3309
+ bitWidth,
3310
+ bias,
3311
+ column
3312
+ );
3313
+ for (let row = 0; row < count; row++) {
3314
+ if (column[row] >= 0) {
3315
+ needed.add(column[row]);
3316
+ }
2098
3317
  }
2099
3318
  }
3319
+ this._emitProgress("symbol-analysis", position + 1, fields.length);
2100
3320
  });
2101
3321
  return symbolUsage;
2102
3322
  }
@@ -2104,12 +3324,20 @@ var init_QvdFileReader = __esm({
2104
3324
  * Parses the symbol table of the QVD file. This method is part of the parsing process
2105
3325
  * and should not be called directly.
2106
3326
  *
2107
- * @param {Map<string, Set<number>>|null} symbolsToKeep Optional map of field names to symbol indices to keep.
2108
- * If provided, only these symbols will be parsed (two-pass filtering optimization).
2109
- * @param {number|null} maxRows Optional maximum number of rows being loaded (for memory estimation).
3327
+ * A field the caller did not select is skipped whole. Its symbol area is neither scanned nor
3328
+ * parsed - the per-field `Offset` and `Length` say exactly where it is, so there is nothing to
3329
+ * walk past - and that is where field selection earns its keep. The index decode is cheap by
3330
+ * comparison; parsing symbols is not.
3331
+ *
3332
+ * @param {Array<Set<number>>|null} symbolsToKeep Optional set of symbol indices to keep per
3333
+ * selected field, indexed by position. If provided, only these symbols will be parsed
3334
+ * (two-pass filtering optimization).
3335
+ * @param {number} rowsToLoad Rows the read covers, for memory estimation.
3336
+ * @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
3337
+ * that is fewer than the window covers - see `_prepare`.
2110
3338
  */
2111
- async _parseSymbolTable(symbolsToKeep = null, maxRows = null) {
2112
- if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset) {
3339
+ async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
3340
+ if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
2113
3341
  throw new QvdCorruptedError(
2114
3342
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
2115
3343
  {
@@ -2118,7 +3346,8 @@ var init_QvdFileReader = __esm({
2118
3346
  }
2119
3347
  );
2120
3348
  }
2121
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
3349
+ const allFields = this._allFields;
3350
+ const fields = this._selectedFields;
2122
3351
  const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
2123
3352
  const symbolTableSize = symbolBuffer.length;
2124
3353
  const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
@@ -2126,51 +3355,34 @@ var init_QvdFileReader = __esm({
2126
3355
  if (this._headerMatchesFile) {
2127
3356
  validateMemoryAvailability(
2128
3357
  symbolTableSize,
2129
- maxRows,
3358
+ rowsToLoad,
2130
3359
  totalRows,
2131
3360
  this._path,
2132
3361
  this._memorySafetyFactor,
2133
- Array.isArray(fields) ? fields.length : 1,
2134
- this._materialisesRows
3362
+ fields.length,
3363
+ this._materialisesRows,
3364
+ liveRows
2135
3365
  );
2136
3366
  }
2137
- warnLargeSymbolTable(
2138
- symbolTableSize,
2139
- maxRows,
2140
- totalRows,
2141
- Array.isArray(fields) ? fields.length : 1,
2142
- this._materialisesRows
2143
- );
2144
- if (!Array.isArray(fields)) {
2145
- fields = [fields];
2146
- }
2147
- for (const field of fields) {
3367
+ warnLargeSymbolTable(symbolTableSize, rowsToLoad, totalRows, fields.length, this._materialisesRows);
3368
+ for (const field of allFields) {
2148
3369
  validateFieldMetadata(field, symbolBuffer.length, this._path);
2149
3370
  }
2150
- this._symbolTable = fields.map((field) => {
3371
+ this._symbolTable = fields.map((field, position) => {
3372
+ this._throwIfAborted();
2151
3373
  const symbolsOffset = parseInt(field["Offset"], 10);
2152
3374
  const symbolsLength = parseInt(field["Length"], 10);
2153
- const fieldName = field["FieldName"];
2154
- const neededSymbols = symbolsToKeep ? symbolsToKeep.get(fieldName) : null;
2155
- const filteringEnabled = neededSymbols !== null;
2156
- const symbols = [];
2157
- let symbolIndex = 0;
2158
- for (let pointer = symbolsOffset; pointer < symbolsOffset + symbolsLength; pointer++) {
2159
- const typeByte = symbolBuffer[pointer++];
2160
- const shouldKeepSymbol = !filteringEnabled || !!(neededSymbols && neededSymbols.has(symbolIndex));
2161
- const { symbol, bytesRead } = parseSymbol(
2162
- typeByte,
2163
- symbolBuffer,
2164
- pointer,
2165
- symbolBuffer.length,
2166
- fieldName,
2167
- this._path,
2168
- shouldKeepSymbol
2169
- );
2170
- symbols.push(symbol);
2171
- pointer += bytesRead - 1;
2172
- symbolIndex++;
2173
- }
3375
+ const symbols = parseFieldSymbols(
3376
+ symbolBuffer,
3377
+ symbolsOffset,
3378
+ symbolsOffset + symbolsLength,
3379
+ // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
3380
+ // `this._selectedFields`, so position is the one key that cannot collide.
3381
+ symbolsToKeep ? symbolsToKeep[position] : null,
3382
+ field["FieldName"],
3383
+ this._path
3384
+ );
3385
+ this._emitProgress("symbol-table", position + 1, fields.length);
2174
3386
  return symbols;
2175
3387
  });
2176
3388
  }
@@ -2189,13 +3401,18 @@ var init_QvdFileReader = __esm({
2189
3401
  * same for every row, so they are hoisted out of the loop and the inner loop does arithmetic
2190
3402
  * into a typed array and nothing else. Rows are assembled later, once, in `load()`.
2191
3403
  *
2192
- * @param {number|null} maxRows The maximum number of rows to parse. If null, all rows are parsed.
3404
+ * The window is what makes chunked iteration cheap: `decodeIndexColumn` walks records by
3405
+ * `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
3406
+ * and how many iterations run. Nothing about the decoder changed to support it.
3407
+ *
3408
+ * @param {QvdRowWindow} window The rows to decode.
2193
3409
  */
2194
- async _parseIndexTable(maxRows = null) {
2195
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "parseIndexTable");
3410
+ async _parseIndexTable(window) {
3411
+ const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
2196
3412
  this._rowsDecoded = rowsToLoad;
2197
- this._indexColumns = fields.map(
2198
- (field) => decodeIndexColumn(
3413
+ this._indexColumns = fields.map((field, position) => {
3414
+ this._throwIfAborted();
3415
+ const column = decodeIndexColumn(
2199
3416
  indexBuffer,
2200
3417
  recordSize,
2201
3418
  rowsToLoad,
@@ -2203,8 +3420,10 @@ var init_QvdFileReader = __esm({
2203
3420
  parseInt(field["BitWidth"], 10),
2204
3421
  parseInt(field["Bias"], 10),
2205
3422
  new Int32Array(rowsToLoad)
2206
- )
2207
- );
3423
+ );
3424
+ this._emitProgress("index-table", position + 1, fields.length);
3425
+ return column;
3426
+ });
2208
3427
  }
2209
3428
  /**
2210
3429
  * Reads the file's schema and header metadata, without touching the symbol or index tables.
@@ -2221,8 +3440,11 @@ var init_QvdFileReader = __esm({
2221
3440
  * @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
2222
3441
  */
2223
3442
  async loadMetadata() {
2224
- await this._readData(null, true);
3443
+ await this._readData({ offset: 0, limit: null }, true);
3444
+ this._emitProgress("header", 0, 1);
2225
3445
  await this._parseHeader();
3446
+ this._emitProgress("header", 1, 1);
3447
+ this._throwIfAborted();
2226
3448
  assert2(this._header, "The QVD file header has not been parsed.");
2227
3449
  const header = this._header["QvdTableHeader"];
2228
3450
  let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
@@ -2257,137 +3479,337 @@ var init_QvdFileReader = __esm({
2257
3479
  /**
2258
3480
  * Loads the QVD file into memory and parses it.
2259
3481
  *
2260
- * @param {number|null} maxRows The maximum number of rows to load. If null, all rows are loaded.
2261
- * Must be a non-negative integer when given.
2262
- * @throws {QvdValidationError} If maxRows is neither null nor a non-negative integer.
3482
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
3483
+ * The rows to load. A number or null means what it always meant - the first N rows, or all of
3484
+ * them - and `{offset, limit}` is the same thing said more precisely, so `5` and
3485
+ * `{offset: 0, limit: 5}` are one read. `maxRows` is accepted as a second name for `limit`.
3486
+ * @throws {QvdValidationError} If the window is not a non-negative integer, null, or a valid
3487
+ * `{offset, limit}` object.
2263
3488
  * @return {Promise<QvdDataFrame>} The loaded QVD file.
2264
3489
  */
2265
- async load(maxRows = null) {
2266
- const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
3490
+ async load(window = null) {
3491
+ const rows = normaliseWindow(window, this._path);
3492
+ const prepared = await this._prepare(rows);
3493
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
3494
+ const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
3495
+ return new QvdDataFrame(
3496
+ data,
3497
+ prepared.columns,
3498
+ prepared.metadata,
3499
+ {
3500
+ ...prepared.loadStats,
3501
+ rowsLoaded: data.length
3502
+ },
3503
+ prepared.storedSymbols
3504
+ );
3505
+ }
3506
+ /**
3507
+ * Reads the file as columns, without ever materialising rows.
3508
+ *
3509
+ * Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
3510
+ * What it keeps instead is what the decoder already produced: one `Int32Array` of stored
3511
+ * indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
3512
+ * fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
3513
+ * bytes per row rather than a boxed value per cell, and the symbols are a few thousand
3514
+ * entries shared across every row that uses them.
3515
+ *
3516
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
3517
+ * The rows to decode, in the same spellings `load()` accepts.
3518
+ * @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
3519
+ */
3520
+ async loadColumnar(window = null) {
3521
+ const rows = normaliseWindow(window, this._path);
3522
+ const prepared = await this._prepare(rows, null, true);
3523
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
3524
+ const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
2267
3525
  assert2(this._indexColumns, "The QVD file index table has not been parsed.");
2268
- const indexColumns = this._indexColumns;
2269
- const fieldCount = indexColumns.length;
2270
- const data = new Array(this._rowsDecoded);
2271
- for (let row = 0; row < this._rowsDecoded; row++) {
2272
- const values = new Array(fieldCount);
2273
- for (let field = 0; field < fieldCount; field++) {
2274
- const symbolIndex = indexColumns[field][row];
2275
- values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
2276
- }
2277
- data[row] = values;
2278
- }
2279
- loadStats.rowsLoaded = data.length;
2280
- return new QvdDataFrame(data, columns, metadata, loadStats);
3526
+ return new QvdColumnTable2({
3527
+ columns: prepared.columns,
3528
+ codesByField: this._indexColumns,
3529
+ symbolsByField: prepared.resolvedByField,
3530
+ halvesByField: prepared.halvesByField,
3531
+ rowCount: this._rowsDecoded,
3532
+ metadata: prepared.metadata,
3533
+ storedSymbols: prepared.storedSymbols,
3534
+ loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
3535
+ });
2281
3536
  }
2282
3537
  /**
2283
- * Reads and decodes the file, stopping short of building rows.
3538
+ * Yields the window as data frames of at most `chunkSize` rows.
2284
3539
  *
2285
- * Everything `load()` and `loadColumnar()` have in common, which is everything except the
2286
- * shape of the answer. Two read paths for one binary format is the drift risk #113 is the
2287
- * standing example of - a stored index resolved one way here and another way there returns
2288
- * plausible wrong values and throws nothing - so there is one path, and the two entry points
2289
- * differ only in what they do with what it returns.
3540
+ * The file is opened, read and parsed **once**; only the index decode and the row building
3541
+ * happen per chunk. That is the whole reason this exists as a method rather than as a loop of
3542
+ * `load({offset, limit})` calls at the call site: the symbol table has to be parsed in full
3543
+ * whatever the chunk size - a stored index in the last chunk can address the first symbol -
3544
+ * and re-parsing it per chunk is what makes the obvious implementation cost more than a plain
3545
+ * load rather than less. PyQvd's chunked read does re-read it, and the comment on #140 records
3546
+ * that as a limitation rather than a design.
2290
3547
  *
2291
- * @param {number|null} maxRows Maximum rows to decode, or null for all of them.
2292
- * @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
2293
- * resolvedByField: Array<Array<any>>}>} The decoded file.
2294
- * @private
3548
+ * What it bounds is row materialisation, which is what actually dominates a large read's heap.
3549
+ * Two chunks of rows are alive at a time, not one - `for await` keeps the yielded frame
3550
+ * reachable while this generator builds the next - which is why `liveRows` below is
3551
+ * `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
3552
+ *
3553
+ * A window covering no rows yields nothing at all, rather than one empty frame - so
3554
+ * `for await` over an exhausted offset does nothing, which is what a paging loop wants.
3555
+ *
3556
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
3557
+ * The rows to cover, in the same spellings `load()` accepts.
3558
+ * @param {number} chunkSize Rows per frame. Must be a positive integer.
3559
+ * @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
2295
3560
  */
2296
- async _decode(maxRows = null) {
2297
- if (maxRows !== null && (typeof maxRows !== "number" || !Number.isInteger(maxRows) || maxRows < 0)) {
2298
- throw new QvdValidationError("maxRows must be a non-negative integer, or null to load all rows", {
2299
- provided: maxRows,
2300
- type: typeof maxRows,
3561
+ async *iterateRows(window, chunkSize) {
3562
+ if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
3563
+ throw new QvdValidationError("chunkSize must be a positive integer", {
3564
+ provided: chunkSize,
3565
+ type: typeof chunkSize,
2301
3566
  file: this._path
2302
3567
  });
2303
3568
  }
2304
- await this._readData(maxRows);
3569
+ const liveRows = { rows: chunkSize * 2, perChunk: 2 };
3570
+ const rows = normaliseWindow(window, this._path);
3571
+ const prepared = await this._prepare(rows, liveRows);
3572
+ for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
3573
+ this._throwIfAborted();
3574
+ const count = Math.min(chunkSize, prepared.rowsAvailable - done);
3575
+ const offset = prepared.offset + done;
3576
+ await this._parseIndexTable({ offset, limit: count });
3577
+ const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
3578
+ yield new QvdDataFrame(
3579
+ data,
3580
+ prepared.columns,
3581
+ prepared.metadata,
3582
+ {
3583
+ ...prepared.loadStats,
3584
+ offset,
3585
+ rowsLoaded: data.length
3586
+ },
3587
+ prepared.storedSymbols
3588
+ );
3589
+ }
3590
+ }
3591
+ /**
3592
+ * Reads the file and resolves its symbols, stopping short of decoding any rows.
3593
+ *
3594
+ * Everything `load()`, `loadColumnar()` and `iterateRows()` have in common, which is everything
3595
+ * that depends on the file rather than on the window. Two read paths for one binary format is
3596
+ * the drift risk #113 is the standing example of - a stored index resolved one way here and
3597
+ * another way there returns plausible wrong values and throws nothing - so there is one path,
3598
+ * and the entry points differ only in what they do with what it returns and how many rows they
3599
+ * ask for at a time.
3600
+ *
3601
+ * @param {QvdRowWindow} window The rows the read covers.
3602
+ * @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
3603
+ * is fewer than the window covers, and how many of them one row of the caller's chunk size
3604
+ * accounts for. Only `iterateRows` passes it; every other read holds what it covers.
3605
+ * @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
3606
+ * cells do not show them, which only a columnar read has a use for.
3607
+ * @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
3608
+ * resolvedByField: Array<Array<any>>,
3609
+ * halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
3610
+ * storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
3611
+ * rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
3612
+ * against it.
3613
+ * @private
3614
+ */
3615
+ async _prepare(window, liveRows = null, wantHalves = false) {
3616
+ this._throwIfAborted();
3617
+ await this._readData(window, false, liveRows);
3618
+ this._emitProgress("header", 0, 1);
2305
3619
  await this._parseHeader();
3620
+ this._emitProgress("header", 1, 1);
3621
+ this._throwIfAborted();
3622
+ assert2(this._header, "The QVD file header has not been parsed.");
3623
+ const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3624
+ const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3625
+ const resolved = resolveWindow(window, totalRows);
3626
+ const rowsAvailable = resolved.limit;
2306
3627
  let symbolsToKeep = null;
2307
3628
  let symbolsKept = null;
2308
- if (maxRows !== null && this._header) {
2309
- const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3629
+ if (window.limit !== null || window.offset > 0) {
2310
3630
  if (symbolTableLength > this._symbolFilteringThreshold) {
2311
- symbolsToKeep = await this._analyzeIndexTableSymbolUsage(maxRows);
2312
- symbolsKept = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
3631
+ symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
3632
+ symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
2313
3633
  }
2314
3634
  }
2315
- await this._parseSymbolTable(symbolsToKeep, maxRows);
2316
- await this._parseIndexTable(maxRows);
2317
- assert2(this._header, "The QVD file header has not been parsed.");
3635
+ await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
2318
3636
  assert2(this._symbolTable, "The QVD file symbol table has not been parsed.");
2319
- assert2(this._indexColumns, "The QVD file index table has not been parsed.");
2320
- const resolvedByField = this._symbolTable.map((symbols) => {
2321
- const resolved = new Array(symbols.length);
2322
- for (let index = 0; index < symbols.length; index++) {
2323
- const value = symbols[index]?.toPrimaryValue();
2324
- resolved[index] = typeof value === "string" && value.trim() !== "" && !isNaN(Number(value)) ? Number(value) : value;
3637
+ this._throwIfAborted();
3638
+ assert2(this._selectedFields, "The QVD file fields have not been resolved.");
3639
+ const resolvedByField = [];
3640
+ const halvesByField = [];
3641
+ const entries = [];
3642
+ this._symbolTable.forEach((symbols, position) => {
3643
+ const { values, entry, halves } = resolveFieldSymbols(
3644
+ symbols,
3645
+ // @ts-ignore - asserted above
3646
+ this._selectedFields[position]["FieldName"],
3647
+ this._duals,
3648
+ this._coerceNumericStrings,
3649
+ wantHalves
3650
+ );
3651
+ resolvedByField.push(values);
3652
+ halvesByField.push(halves);
3653
+ if (entry !== null) {
3654
+ entries.push(entry);
2325
3655
  }
2326
- return resolved;
2327
3656
  });
2328
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
2329
- if (!Array.isArray(fields)) {
2330
- fields = [fields];
2331
- }
2332
- const columns = fields.map((field) => field["FieldName"]);
3657
+ const columns = this._selectedFields.map((field) => field["FieldName"]);
2333
3658
  const metadata = this._header["QvdTableHeader"];
3659
+ const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
3660
+ if (storedSymbols !== null) {
3661
+ attachStoredSymbols(metadata, storedSymbols);
3662
+ }
2334
3663
  const loadStats = {
2335
- symbolTableBytes: parseInt(this._header["QvdTableHeader"]["Offset"], 10),
2336
- totalRows: parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10),
2337
- rowsLoaded: this._rowsDecoded,
3664
+ symbolTableBytes: symbolTableLength,
3665
+ totalRows,
3666
+ rowsLoaded: 0,
3667
+ offset: resolved.offset,
2338
3668
  symbolFiltering: symbolsToKeep !== null,
2339
3669
  symbolsKept
2340
3670
  };
2341
- return { columns, metadata, loadStats, resolvedByField };
3671
+ return {
3672
+ columns,
3673
+ metadata,
3674
+ loadStats,
3675
+ resolvedByField,
3676
+ halvesByField,
3677
+ storedSymbols,
3678
+ rowsAvailable,
3679
+ offset: resolved.offset
3680
+ };
2342
3681
  }
2343
3682
  /**
2344
- * Reads the file as columns, without ever materialising rows.
3683
+ * Builds rows from the columns currently decoded.
2345
3684
  *
2346
- * Shares every step with `load()` up to the point where rows would be built - see `_decode`.
2347
- * What it keeps instead is what the decoder already produced: one `Int32Array` of stored
2348
- * indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
2349
- * fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
2350
- * bytes per row rather than a boxed value per cell, and the symbols are a few thousand
2351
- * entries shared across every row that uses them.
3685
+ * `data` stays eager: of the four ways this library is used - a full read, a preview already
3686
+ * bounded by a limit, writing an array out, and reading metadata - not one is helped by
3687
+ * materialising a row only when it is touched, and a lazy accessor would cost a proxy, a cache
3688
+ * and mutation semantics to serve none of them. A caller who wants columns without paying for
3689
+ * rows uses `QvdColumnTable`, which stops before this loop.
2352
3690
  *
2353
- * @param {number|null} maxRows The maximum number of rows to decode.
2354
- * @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
3691
+ * @param {Array<Array<any>>} resolvedByField One resolved value per distinct symbol, per field.
3692
+ * @param {number} progressBase Rows already delivered before this call, so that progress over a
3693
+ * chunked iteration counts the whole window rather than restarting at every chunk.
3694
+ * @param {number} progressTotal Rows the whole window covers.
3695
+ * @return {Array<Array<any>>} The rows.
3696
+ * @private
2355
3697
  */
2356
- async loadColumnar(maxRows = null) {
2357
- const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
2358
- const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
3698
+ _buildRows(resolvedByField, progressBase, progressTotal) {
2359
3699
  assert2(this._indexColumns, "The QVD file index table has not been parsed.");
2360
- return new QvdColumnTable2({
2361
- columns,
2362
- codesByField: this._indexColumns,
2363
- symbolsByField: resolvedByField,
2364
- rowCount: this._rowsDecoded,
2365
- metadata,
2366
- loadStats
2367
- });
3700
+ const indexColumns = this._indexColumns;
3701
+ const fieldCount = indexColumns.length;
3702
+ const rowCount = this._rowsDecoded;
3703
+ const data = new Array(rowCount);
3704
+ const reportInterval = Math.max(1, Math.floor(progressTotal / 100));
3705
+ for (let row = 0; row < rowCount; row++) {
3706
+ const values = new Array(fieldCount);
3707
+ for (let field = 0; field < fieldCount; field++) {
3708
+ const symbolIndex = indexColumns[field][row];
3709
+ values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
3710
+ }
3711
+ data[row] = values;
3712
+ if ((progressBase + row + 1) % reportInterval === 0 || row + 1 === rowCount) {
3713
+ this._throwIfAborted();
3714
+ this._emitProgress("rows", progressBase + row + 1, progressTotal);
3715
+ }
3716
+ }
3717
+ return data;
2368
3718
  }
2369
3719
  };
2370
3720
  }
2371
3721
  });
2372
3722
 
2373
3723
  // src/QvdDataFrame.js
3724
+ function defaultFieldHeader(fieldName) {
3725
+ return {
3726
+ FieldName: fieldName,
3727
+ BitOffset: 0,
3728
+ BitWidth: 0,
3729
+ Bias: 0,
3730
+ NoOfSymbols: 0,
3731
+ Offset: 0,
3732
+ Length: 0,
3733
+ Comment: "",
3734
+ NumberFormat: {
3735
+ Type: "UNKNOWN",
3736
+ nDec: "0",
3737
+ UseThou: "0",
3738
+ Fmt: "",
3739
+ Dec: "",
3740
+ Thou: ""
3741
+ },
3742
+ Tags: {}
3743
+ };
3744
+ }
3745
+ function defaultHeader(columns) {
3746
+ return {
3747
+ QvBuildNo: 50667,
3748
+ CreatorDoc: "",
3749
+ CreateUtcTime: "",
3750
+ SourceCreateUtcTime: "",
3751
+ SourceFileUtcTime: "",
3752
+ SourceFileSize: -1,
3753
+ StaleUtcTime: "",
3754
+ TableName: "",
3755
+ Fields: {
3756
+ QvdFieldHeader: columns.map(defaultFieldHeader)
3757
+ },
3758
+ NoOfRecords: 0,
3759
+ RecordByteSize: 0,
3760
+ Offset: 0,
3761
+ Length: 0,
3762
+ Compression: "",
3763
+ Comment: "",
3764
+ EncryptionInfo: "",
3765
+ TableTags: "",
3766
+ ProfilingData: "",
3767
+ Lineage: {}
3768
+ };
3769
+ }
2374
3770
  var QvdDataFrame;
2375
3771
  var init_QvdDataFrame = __esm({
2376
3772
  "src/QvdDataFrame.js"() {
2377
3773
  init_QvdErrors();
3774
+ init_cellRules();
3775
+ init_readOptions();
3776
+ init_storedSymbols();
3777
+ __name(defaultFieldHeader, "defaultFieldHeader");
3778
+ __name(defaultHeader, "defaultHeader");
2378
3779
  QvdDataFrame = class _QvdDataFrame {
3780
+ static {
3781
+ __name(this, "QvdDataFrame");
3782
+ }
2379
3783
  /**
2380
3784
  * Represents the data frame stored inside a QVD file.
3785
+ *
3786
+ * The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
3787
+ * its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
3788
+ * Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
3789
+ * holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
3790
+ * header's record did for a frame built from some of a read's columns.
3791
+ *
2381
3792
  * @param {Array<Array<any>>} data The data of the data frame.
2382
3793
  * @param {Array<string>} columns The columns of the data frame.
2383
3794
  * @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
2384
3795
  * @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
3796
+ * @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
3797
+ * shows only one half of its symbol (optional) - see `storedSymbols`.
3798
+ * @throws {QvdValidationError} If the record is malformed.
2385
3799
  */
2386
- constructor(data, columns, metadata = null, loadStats = null) {
3800
+ constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
2387
3801
  this._data = data;
2388
3802
  this._columns = columns;
2389
3803
  this._metadata = metadata;
3804
+ this._ownsMetadata = false;
2390
3805
  this._loadStats = loadStats;
3806
+ this._storedSymbols = narrowStoredSymbols(
3807
+ normaliseStoredSymbols(
3808
+ // @ts-ignore - a symbol-keyed property the reader defines on the header object
3809
+ storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
3810
+ ),
3811
+ columns
3812
+ );
2391
3813
  }
2392
3814
  /**
2393
3815
  * Returns the data of the data frame.
@@ -2414,15 +3836,40 @@ var init_QvdDataFrame = __esm({
2414
3836
  get metadata() {
2415
3837
  return this._metadata;
2416
3838
  }
3839
+ /**
3840
+ * What the frame's cells were read from, where a cell shows only one half of its symbol.
3841
+ *
3842
+ * A dual read as its number has a text the cell does not show; one read as its text has a number;
3843
+ * a string read as a number has the text it was spelled with. The record keeps those halves, per
3844
+ * field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
3845
+ * and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
3846
+ * `select`, `toDict` and `fromDict`.
3847
+ *
3848
+ * Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
3849
+ * the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
3850
+ * pure number.
3851
+ *
3852
+ * @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
3853
+ * showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
3854
+ * in the record it was given or found - one from `select`, or one built from some of a read's
3855
+ * columns and its header - has an empty record rather than null, because a frame given null takes
3856
+ * the record its header carries, which describes every field of the read.
3857
+ */
3858
+ get storedSymbols() {
3859
+ return this._storedSymbols;
3860
+ }
2417
3861
  /**
2418
3862
  * Returns statistics about the read that produced this data frame.
2419
3863
  *
2420
- * Only a frame returned by fromQvd() carries these; fromDict(), head() and tail() produce
2421
- * frames that describe no particular read, and report null rather than a stale figure.
3864
+ * Carried by every frame that came from a file - `fromQvd()`, and each chunk `iterate()` yields,
3865
+ * which is how a chunk reports its `offset`. `fromDict()`, `head()`, `tail()`, `rows()` and
3866
+ * `select()` describe no particular read and report null rather than a stale figure.
2422
3867
  *
2423
3868
  * The main use is confirming that a lazy load actually filtered the symbol table:
2424
3869
  * `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
2425
- * survived it, which for a small maxRows should be a tiny fraction of the file's total.
3870
+ * survived it. Note that a bounded read does not filter on its own - the two-pass path engages
3871
+ * only above `symbolFilteringThreshold`, so on a file below it this reports false and every
3872
+ * symbol was parsed however few rows were asked for.
2426
3873
  *
2427
3874
  * @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
2428
3875
  */
@@ -2531,54 +3978,40 @@ var init_QvdDataFrame = __esm({
2531
3978
  * @property {string} [profilingData] - Profiling data
2532
3979
  * @property {Object|string} [lineage] - Lineage
2533
3980
  */
3981
+ /**
3982
+ * The header a metadata setter may change: this frame's own.
3983
+ *
3984
+ * A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
3985
+ * `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
3986
+ * change copies it, and a change made through one frame never reaches another. A frame with no header
3987
+ * gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
3988
+ * so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
3989
+ *
3990
+ * @return {any} The header.
3991
+ */
3992
+ _ownMetadata() {
3993
+ if (!this._metadata) {
3994
+ this._metadata = defaultHeader(this._columns);
3995
+ } else if (!this._ownsMetadata) {
3996
+ const record = this._metadata[STORED_SYMBOLS];
3997
+ this._metadata = structuredClone(this._metadata);
3998
+ if (record) {
3999
+ attachStoredSymbols(this._metadata, record);
4000
+ }
4001
+ }
4002
+ this._ownsMetadata = true;
4003
+ return this._metadata;
4004
+ }
2534
4005
  /**
2535
4006
  * Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
4007
+ *
4008
+ * The change applies to this frame only, never to a frame it was derived from or shares a header
4009
+ * with.
4010
+ *
2536
4011
  * @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
2537
4012
  */
2538
4013
  setFileMetadata(metadata) {
2539
- if (!this._metadata) {
2540
- this._metadata = {
2541
- QvBuildNo: 50667,
2542
- CreatorDoc: "",
2543
- CreateUtcTime: "",
2544
- SourceCreateUtcTime: "",
2545
- SourceFileUtcTime: "",
2546
- SourceFileSize: -1,
2547
- StaleUtcTime: "",
2548
- TableName: "",
2549
- Fields: {
2550
- QvdFieldHeader: this._columns.map((column) => ({
2551
- FieldName: column,
2552
- BitOffset: 0,
2553
- BitWidth: 0,
2554
- Bias: 0,
2555
- NoOfSymbols: 0,
2556
- Offset: 0,
2557
- Length: 0,
2558
- Comment: "",
2559
- NumberFormat: {
2560
- Type: "UNKNOWN",
2561
- nDec: "0",
2562
- UseThou: "0",
2563
- Fmt: "",
2564
- Dec: "",
2565
- Thou: ""
2566
- },
2567
- Tags: {}
2568
- }))
2569
- },
2570
- NoOfRecords: 0,
2571
- RecordByteSize: 0,
2572
- Offset: 0,
2573
- Length: 0,
2574
- Compression: "",
2575
- Comment: "",
2576
- EncryptionInfo: "",
2577
- TableTags: "",
2578
- ProfilingData: "",
2579
- Lineage: {}
2580
- };
2581
- }
4014
+ const header = this._ownMetadata();
2582
4015
  const modifiableFields = [
2583
4016
  "qvBuildNo",
2584
4017
  "creatorDoc",
@@ -2613,7 +4046,7 @@ var init_QvdDataFrame = __esm({
2613
4046
  };
2614
4047
  modifiableFields.forEach((field) => {
2615
4048
  if (metadata[field] !== void 0) {
2616
- this._metadata[fieldMapping[field]] = metadata[field];
4049
+ header[fieldMapping[field]] = metadata[field];
2617
4050
  }
2618
4051
  });
2619
4052
  }
@@ -2626,30 +4059,44 @@ var init_QvdDataFrame = __esm({
2626
4059
  /**
2627
4060
  * Sets modifiable field-level metadata for a specific field.
2628
4061
  * Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
4062
+ *
4063
+ * Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
4064
+ * header does not describe. The change applies to this frame only, never to a frame it was derived
4065
+ * from or shares a header with.
4066
+ *
2629
4067
  * @param {string} fieldName The name of the field.
2630
4068
  * @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
4069
+ * @throws {QvdValidationError} If the frame has no column of that name.
2631
4070
  */
2632
4071
  setFieldMetadata(fieldName, metadata) {
2633
- if (!this._metadata || !this._metadata.Fields || !this._metadata.Fields.QvdFieldHeader) {
2634
- return;
4072
+ if (!this._columns.includes(fieldName)) {
4073
+ throw new QvdValidationError(`Column '${fieldName}' does not exist`, {
4074
+ column: fieldName,
4075
+ availableColumns: this._columns
4076
+ });
2635
4077
  }
2636
- let fields = this._metadata.Fields.QvdFieldHeader;
4078
+ const header = this._ownMetadata();
4079
+ if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
4080
+ header.Fields = { QvdFieldHeader: [] };
4081
+ }
4082
+ let fields = header.Fields.QvdFieldHeader;
2637
4083
  if (!Array.isArray(fields)) {
2638
4084
  fields = [fields];
2639
- this._metadata.Fields.QvdFieldHeader = fields;
4085
+ header.Fields.QvdFieldHeader = fields;
2640
4086
  }
2641
- const fieldIndex = fields.findIndex((f) => f.FieldName === fieldName);
2642
- if (fieldIndex === -1) {
2643
- return;
4087
+ let field = fields.find((f) => f.FieldName === fieldName);
4088
+ if (!field) {
4089
+ field = defaultFieldHeader(fieldName);
4090
+ fields.push(field);
2644
4091
  }
2645
4092
  if (metadata.comment !== void 0) {
2646
- fields[fieldIndex].Comment = metadata.comment;
4093
+ field.Comment = metadata.comment;
2647
4094
  }
2648
4095
  if (metadata.numberFormat !== void 0) {
2649
- fields[fieldIndex].NumberFormat = metadata.numberFormat;
4096
+ field.NumberFormat = metadata.numberFormat;
2650
4097
  }
2651
4098
  if (metadata.tags !== void 0) {
2652
- fields[fieldIndex].Tags = metadata.tags;
4099
+ field.Tags = metadata.tags;
2653
4100
  }
2654
4101
  }
2655
4102
  /**
@@ -2666,7 +4113,7 @@ var init_QvdDataFrame = __esm({
2666
4113
  type: typeof n
2667
4114
  });
2668
4115
  }
2669
- return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
4116
+ return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
2670
4117
  }
2671
4118
  /**
2672
4119
  * Returns the last n rows of the data frame.
@@ -2682,7 +4129,13 @@ var init_QvdDataFrame = __esm({
2682
4129
  type: typeof n
2683
4130
  });
2684
4131
  }
2685
- return new _QvdDataFrame(n === 0 ? [] : this._data.slice(-n), this._columns, this._metadata);
4132
+ return new _QvdDataFrame(
4133
+ n === 0 ? [] : this._data.slice(-n),
4134
+ this._columns,
4135
+ this._metadata,
4136
+ null,
4137
+ this._storedSymbols
4138
+ );
2686
4139
  }
2687
4140
  /**
2688
4141
  * Returns the selected rows of the data frame.
@@ -2710,7 +4163,9 @@ var init_QvdDataFrame = __esm({
2710
4163
  return new _QvdDataFrame(
2711
4164
  args.map((index) => this._data[index]),
2712
4165
  this._columns,
2713
- this._metadata
4166
+ this._metadata,
4167
+ null,
4168
+ this._storedSymbols
2714
4169
  );
2715
4170
  }
2716
4171
  /**
@@ -2722,6 +4177,50 @@ var init_QvdDataFrame = __esm({
2722
4177
  * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
2723
4178
  */
2724
4179
  at(row, column) {
4180
+ const index = this._cellIndex(row, column);
4181
+ return this._data[row][index];
4182
+ }
4183
+ /**
4184
+ * Returns the text of the value at the specified row and column.
4185
+ *
4186
+ * The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
4187
+ * `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
4188
+ * or the string it was spelled as - and is null for a number that was stored as a pure number.
4189
+ *
4190
+ * ```js
4191
+ * const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
4192
+ * df.at(0, 'date'); // -52593
4193
+ * df.textAt(0, 'date'); // '1756-01-01'
4194
+ * ```
4195
+ *
4196
+ * @param {number} row The index of the row.
4197
+ * @param {string} column The name of the column.
4198
+ * @return {string|null} The text, or null for NULL and for a number with no text.
4199
+ * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
4200
+ */
4201
+ textAt(row, column) {
4202
+ const index = this._cellIndex(row, column);
4203
+ const value = this._data[row][index];
4204
+ if (typeof value === "string") {
4205
+ return value;
4206
+ }
4207
+ if (typeof value === "number") {
4208
+ const entry = storedSymbolsEntry(this._storedSymbols, column);
4209
+ return entry === null ? null : storedTextOf(entry, value);
4210
+ }
4211
+ const dual = asDual(value);
4212
+ return dual !== null && typeof dual.text === "string" ? dual.text : null;
4213
+ }
4214
+ /**
4215
+ * Checks a row and a column name, and returns the column's position.
4216
+ *
4217
+ * @param {number} row The index of the row.
4218
+ * @param {string} column The name of the column.
4219
+ * @return {number} The column's position.
4220
+ * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
4221
+ * @private
4222
+ */
4223
+ _cellIndex(row, column) {
2725
4224
  if (typeof row !== "number" || !Number.isInteger(row)) {
2726
4225
  throw new QvdValidationError("Row index must be an integer", {
2727
4226
  provided: row,
@@ -2741,7 +4240,7 @@ var init_QvdDataFrame = __esm({
2741
4240
  availableColumns: this._columns
2742
4241
  });
2743
4242
  }
2744
- return this._data[row][this._columns.indexOf(column)];
4243
+ return this._columns.indexOf(column);
2745
4244
  }
2746
4245
  /**
2747
4246
  * Selects the specified columns from the data frame.
@@ -2762,15 +4261,34 @@ var init_QvdDataFrame = __esm({
2762
4261
  const indices = args.map((arg) => this._columns.indexOf(arg));
2763
4262
  const data = this._data.map((row) => indices.map((index) => row[index]));
2764
4263
  const columns = indices.map((index) => this._columns[index]);
2765
- return new _QvdDataFrame(data, columns, this._metadata);
4264
+ return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
2766
4265
  }
2767
4266
  /**
2768
4267
  * Returns the data frame as a dictionary.
2769
4268
  *
2770
- * @return {Promise<{columns: Array<string>, data: Array<Array<any>>}>} The data frame as a dictionary.
4269
+ * Everything a frame needs to write the same file again, as plain data: the header, and the
4270
+ * stored-symbol record that says what cells showing one half of a symbol were read from. So
4271
+ * `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
4272
+ * `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
4273
+ *
4274
+ * @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
2771
4275
  */
2772
4276
  async toDict() {
2773
- return { columns: this._columns, data: this._data };
4277
+ return this.toJSON();
4278
+ }
4279
+ /**
4280
+ * The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
4281
+ * `fromDict(JSON.parse(...))` can revive.
4282
+ *
4283
+ * @return {QvdDataFrameDict} The data frame as a dictionary.
4284
+ */
4285
+ toJSON() {
4286
+ return {
4287
+ columns: this._columns,
4288
+ data: this._data,
4289
+ metadata: this._metadata,
4290
+ storedSymbols: this._storedSymbols
4291
+ };
2774
4292
  }
2775
4293
  /**
2776
4294
  * Persists the data frame to a QVD file.
@@ -2799,6 +4317,33 @@ var init_QvdDataFrame = __esm({
2799
4317
  * @param {Object} [options] Optional loading options.
2800
4318
  * @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
2801
4319
  * integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
4320
+ * This is the older name for `limit`; the two are the same option and passing both throws.
4321
+ * @param {number|null} [options.limit] Rows to read, counting from `offset`. The same number as
4322
+ * `maxRows`, spelled so that it reads correctly beside an offset.
4323
+ * @param {number} [options.offset=0] File row to start at. An offset past the end of the file
4324
+ * returns no rows rather than throwing, so a paging loop terminates on its own.
4325
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
4326
+ * appear in the result. Unselected fields have their symbols skipped entirely rather than
4327
+ * parsed and discarded. An unknown or repeated name throws.
4328
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
4329
+ * text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
4330
+ * `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
4331
+ * `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
4332
+ * implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
4333
+ * `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
4334
+ * double, a string and NULL read the same in every mode: a number, a string and null. Anything else
4335
+ * throws.
4336
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
4337
+ * reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
4338
+ * reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
4339
+ * `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
4340
+ * `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
4341
+ * original string or dual back, and a value that two stored values read as is refused there
4342
+ * rather than written as either. Anything but a boolean throws.
4343
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
4344
+ * read proceeds - the same shape `toQvd`'s callback receives.
4345
+ * @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
4346
+ * which is a `DOMException` named `AbortError` unless you aborted with a reason of your own.
2802
4347
  * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
2803
4348
  * must be within this directory, with symlinks resolved first, so a link inside it that points
2804
4349
  * outside it is rejected. Defaults to the current working directory. To permit an entire
@@ -2811,17 +4356,64 @@ var init_QvdDataFrame = __esm({
2811
4356
  * **Zero disables the memory check entirely.**
2812
4357
  * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
2813
4358
  * a lazy load switches to the two-pass filtering path. Defaults to 50MB.
2814
- * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
4359
+ * @throws {QvdValidationError} If a window option is not a non-negative integer, if both
4360
+ * `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
4361
+ * is not one of its modes, or if `coerceNumericStrings` is not a boolean.
2815
4362
  * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
2816
4363
  */
2817
4364
  static async fromQvd(path3, options = {}) {
2818
4365
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
2819
- const readerOptions = {
2820
- allowedDir: options.allowedDir,
2821
- memorySafetyFactor: options.memorySafetyFactor,
2822
- symbolFilteringThreshold: options.symbolFilteringThreshold
2823
- };
2824
- return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
4366
+ return await new QvdFileReader2(path3, readerOptionsFrom(options)).load(windowFrom(options));
4367
+ }
4368
+ /**
4369
+ * Reads a QVD file in chunks, as an async generator of data frames.
4370
+ *
4371
+ * The file is opened, read and parsed once; only the index decode and the row building happen
4372
+ * per chunk, so what this bounds is row materialisation - the part that actually dominates a
4373
+ * large read's heap. It is **not** constant-memory reading of an arbitrarily large file: the
4374
+ * symbol table is parsed in full whatever the chunk size, because a stored index in the last
4375
+ * chunk can address the first symbol. On a high-cardinality file that table is the bulk of the
4376
+ * cost, and `readMetadata` is the only read that avoids it.
4377
+ *
4378
+ * ```js
4379
+ * for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
4380
+ * process(chunk.data);
4381
+ * }
4382
+ * ```
4383
+ *
4384
+ * A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
4385
+ * run its body.
4386
+ *
4387
+ * Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
4388
+ * them, so a chunk written on its own writes the symbols its cells were read from.
4389
+ *
4390
+ * @param {string} path The path to the QVD file.
4391
+ * @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
4392
+ * @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
4393
+ * @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
4394
+ * @param {number|null} [options.limit] Rows to cover, counting from `offset`.
4395
+ * @param {number} [options.offset=0] File row to start at.
4396
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
4397
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
4398
+ * its text, or a frozen `QvdDual` holding both. Anything else throws.
4399
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
4400
+ * reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
4401
+ * `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
4402
+ * `storedSymbols`. Anything but a boolean throws.
4403
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
4404
+ * over the rows counts the whole window, not each chunk.
4405
+ * @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
4406
+ * @param {string} [options.allowedDir] Directory the path must resolve inside.
4407
+ * @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
4408
+ * charged for two chunks of rows rather than the window. Zero disables the check.
4409
+ * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
4410
+ * windowed read switches to two-pass filtering.
4411
+ * @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
4412
+ */
4413
+ static async *iterate(path3, options = {}) {
4414
+ const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
4415
+ const reader = new QvdFileReader2(path3, readerOptionsFrom(options));
4416
+ yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
2825
4417
  }
2826
4418
  /**
2827
4419
  * Reads a QVD file's schema and header metadata, without reading its data.
@@ -2845,17 +4437,28 @@ var init_QvdDataFrame = __esm({
2845
4437
  * @param {Object} [options] Optional reading options.
2846
4438
  * @param {string} [options.allowedDir] Optional allowed directory path, applied exactly as it
2847
4439
  * is for `fromQvd`.
4440
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`, as on
4441
+ * the reads that return data. Only the `read` and `header` stages occur here; there are no
4442
+ * symbols to parse and no rows to build.
4443
+ * @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
2848
4444
  * @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
2849
4445
  */
2850
4446
  static async readMetadata(path3, options = {}) {
2851
4447
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
2852
- return await new QvdFileReader2(path3, { allowedDir: options.allowedDir }).loadMetadata();
4448
+ return await new QvdFileReader2(path3, metadataOptionsFrom(options)).loadMetadata();
2853
4449
  }
2854
4450
  /**
2855
4451
  * Constructs a data frame from a dictionary.
2856
4452
  *
2857
- * @param {{columns: Array<string>, data: Array<Array<any>>}} data The dictionary to construct the data frame from.
4453
+ * Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
4454
+ * rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
4455
+ * spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
4456
+ * here rather than written.
4457
+ *
4458
+ * @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
2858
4459
  * @return {Promise<QvdDataFrame>} The constructed data frame.
4460
+ * @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
4461
+ * or `storedSymbols` is malformed or names a field that is not one of the columns.
2859
4462
  */
2860
4463
  static async fromDict(data) {
2861
4464
  if (!data.columns) {
@@ -2868,20 +4471,290 @@ var init_QvdDataFrame = __esm({
2868
4471
  data
2869
4472
  });
2870
4473
  }
2871
- return new _QvdDataFrame(data.data, data.columns);
4474
+ const { metadata = null, storedSymbols = null } = data;
4475
+ if (metadata !== null && !isPlainObject(metadata)) {
4476
+ throw new QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
4477
+ type: typeof metadata
4478
+ });
4479
+ }
4480
+ const record = normaliseStoredSymbols(storedSymbols);
4481
+ if (record !== null) {
4482
+ const unknown = record.find((entry) => !data.columns.includes(entry.field));
4483
+ if (unknown !== void 0) {
4484
+ throw new QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
4485
+ field: unknown.field,
4486
+ availableColumns: data.columns
4487
+ });
4488
+ }
4489
+ }
4490
+ return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
2872
4491
  }
2873
4492
  };
2874
4493
  }
2875
4494
  });
2876
4495
 
4496
+ // src/QvdSymbol.js
4497
+ init_QvdErrors();
4498
+ init_cellRules();
4499
+ init_symbolBytes();
4500
+ function checkInteger(value, context) {
4501
+ if (typeof value === "number" && isStoredAsInt(value)) {
4502
+ return;
4503
+ }
4504
+ throw new QvdValidationError(
4505
+ `The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
4506
+ { ...context, half: "integer", type: typeof value }
4507
+ );
4508
+ }
4509
+ __name(checkInteger, "checkInteger");
4510
+ var QvdSymbol = class _QvdSymbol {
4511
+ static {
4512
+ __name(this, "QvdSymbol");
4513
+ }
4514
+ /**
4515
+ * Constructs a new QVD symbol.
4516
+ *
4517
+ * @param {number|null} intValue The integer value.
4518
+ * @param {number|null} doubleValue The double value.
4519
+ * @param {string|null} stringValue The string value.
4520
+ */
4521
+ constructor(intValue, doubleValue, stringValue) {
4522
+ this._intValue = intValue;
4523
+ this._doubleValue = doubleValue;
4524
+ this._stringValue = stringValue;
4525
+ }
4526
+ /**
4527
+ * Returns the integer value of this symbol.
4528
+ *
4529
+ * @return {number|null} The integer value.
4530
+ */
4531
+ get intValue() {
4532
+ return this._intValue;
4533
+ }
4534
+ /**
4535
+ * Returns the double value of this symbol.
4536
+ *
4537
+ * @return {number|null} The double value.
4538
+ */
4539
+ get doubleValue() {
4540
+ return this._doubleValue;
4541
+ }
4542
+ /**
4543
+ * Returns the string value of this symbol.
4544
+ *
4545
+ * @return {string|null} The string value.
4546
+ */
4547
+ get stringValue() {
4548
+ return this._stringValue;
4549
+ }
4550
+ /**
4551
+ * Retrieves the primary value of this symbol. The primary value is descriptive raw value.
4552
+ * It is either the string value, the integer value or the double value, prioritized in this order.
4553
+ *
4554
+ * @return {number|string|null} The primary value.
4555
+ */
4556
+ toPrimaryValue() {
4557
+ if (null != this._stringValue) {
4558
+ return this._stringValue;
4559
+ } else if (null != this._intValue) {
4560
+ return this._intValue;
4561
+ } else if (null != this._doubleValue) {
4562
+ return this._doubleValue;
4563
+ } else {
4564
+ return null;
4565
+ }
4566
+ }
4567
+ /**
4568
+ * Converts the symbol to its byte representation.
4569
+ *
4570
+ * The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
4571
+ * double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
4572
+ * checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
4573
+ * outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
4574
+ * integer and a double silently lost the double, a text with a NUL produced a symbol that ends
4575
+ * early, and a text with an unpaired surrogate was written with U+FFFD in its place.
4576
+ *
4577
+ * A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
4578
+ * `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
4579
+ * `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
4580
+ *
4581
+ * @return {Buffer} The byte representation of the symbol.
4582
+ * @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
4583
+ * holds a half no symbol can store. The message names the half.
4584
+ */
4585
+ toByteRepresentation() {
4586
+ const intValue = this._intValue ?? null;
4587
+ const doubleValue = this._doubleValue ?? null;
4588
+ const stringValue = this._stringValue ?? null;
4589
+ if (intValue !== null && doubleValue !== null) {
4590
+ throw new QvdValidationError("A symbol holds an integer or a double, not both", {
4591
+ intValue,
4592
+ doubleValue
4593
+ });
4594
+ }
4595
+ if (intValue === null && doubleValue === null && stringValue === null) {
4596
+ throw new QvdValidationError("The symbol does not contain any value.", {
4597
+ intValue,
4598
+ doubleValue,
4599
+ stringValue
4600
+ });
4601
+ }
4602
+ if (intValue !== null) {
4603
+ checkInteger(intValue, {});
4604
+ }
4605
+ if (doubleValue !== null) {
4606
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4607
+ }
4608
+ if (stringValue !== null) {
4609
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4610
+ }
4611
+ const number = intValue ?? doubleValue;
4612
+ const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
4613
+ const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
4614
+ writeSymbol(buffer, 0, kind, number, stringValue);
4615
+ return buffer;
4616
+ }
4617
+ /**
4618
+ * Checks if this symbol is equal to another symbol.
4619
+ *
4620
+ * By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
4621
+ * `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
4622
+ * answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
4623
+ * or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
4624
+ * storage kind, so either kind matches.
4625
+ *
4626
+ * @param {*} value The object to compare with.
4627
+ * @return {boolean} True if the objects are equal, false otherwise.
4628
+ */
4629
+ equals(value) {
4630
+ if (value === null || typeof value !== "object") {
4631
+ return false;
4632
+ }
4633
+ const intValue = this._intValue ?? null;
4634
+ const doubleValue = this._doubleValue ?? null;
4635
+ const stringValue = this._stringValue ?? null;
4636
+ const dual = asDual(value);
4637
+ if (dual !== null) {
4638
+ return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
4639
+ }
4640
+ if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
4641
+ return false;
4642
+ }
4643
+ return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
4644
+ }
4645
+ /**
4646
+ * Constructs a pure integer value symbol.
4647
+ *
4648
+ * @param {number} intValue The integer value.
4649
+ * @return {QvdSymbol} The constructed value symbol.
4650
+ * @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
4651
+ */
4652
+ static fromIntValue(intValue) {
4653
+ checkInteger(intValue, {});
4654
+ return new _QvdSymbol(intValue, null, null);
4655
+ }
4656
+ /**
4657
+ * Constructs a pure double value symbol.
4658
+ *
4659
+ * @param {number} doubleValue The double value.
4660
+ * @return {QvdSymbol} The constructed value symbol.
4661
+ * @throws {QvdValidationError} If the double is not a finite number.
4662
+ */
4663
+ static fromDoubleValue(doubleValue) {
4664
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4665
+ return new _QvdSymbol(null, doubleValue, null);
4666
+ }
4667
+ /**
4668
+ * Constructs a pure string value symbol.
4669
+ *
4670
+ * @param {string} stringValue The string value.
4671
+ * @return {QvdSymbol} The constructed value symbol.
4672
+ * @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
4673
+ */
4674
+ static fromStringValue(stringValue) {
4675
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4676
+ return new _QvdSymbol(null, null, stringValue);
4677
+ }
4678
+ /**
4679
+ * Constructs a dual value symbol from an integer and a string value.
4680
+ *
4681
+ * @param {number} intValue The integer value.
4682
+ * @param {string} stringValue The string value.
4683
+ * @return {QvdSymbol} The constructed value symbol.
4684
+ * @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
4685
+ * is not a string, or holds a NUL or an unpaired surrogate.
4686
+ */
4687
+ static fromDualIntValue(intValue, stringValue) {
4688
+ checkInteger(intValue, {});
4689
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4690
+ return new _QvdSymbol(intValue, null, stringValue);
4691
+ }
4692
+ /**
4693
+ * Constructs a dual value symbol from a double and a string value.
4694
+ *
4695
+ * @param {number} doubleValue The double value.
4696
+ * @param {string} stringValue The string value.
4697
+ * @return {QvdSymbol} The constructed value symbol.
4698
+ * @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
4699
+ * or holds a NUL or an unpaired surrogate.
4700
+ */
4701
+ static fromDualDoubleValue(doubleValue, stringValue) {
4702
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4703
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4704
+ return new _QvdSymbol(null, doubleValue, stringValue);
4705
+ }
4706
+ };
4707
+
4708
+ // src/index.js
4709
+ init_QvdDual();
4710
+
4711
+ // src/util/qlikDate.js
4712
+ init_QvdErrors();
4713
+ init_cellRules();
4714
+ var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
4715
+ var MS_PER_DAY = 864e5;
4716
+ var MAX_DATE_MS = 864e13;
4717
+ var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
4718
+ var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
4719
+ function qlikSerialToDate(serial) {
4720
+ const dual = asDual(serial);
4721
+ const number = dual === null ? serial : dual.number;
4722
+ if (typeof number !== "number" || !Number.isFinite(number)) {
4723
+ throw new QvdValidationError("A Qlik date serial must be a finite number", {
4724
+ provided: describeType(number),
4725
+ type: typeof serial
4726
+ });
4727
+ }
4728
+ const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
4729
+ if (!(Math.abs(ms) <= MAX_DATE_MS)) {
4730
+ throw new QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
4731
+ serial: number,
4732
+ minSerial: MIN_SERIAL,
4733
+ maxSerial: MAX_SERIAL
4734
+ });
4735
+ }
4736
+ return new Date(ms);
4737
+ }
4738
+ __name(qlikSerialToDate, "qlikSerialToDate");
4739
+ function dateToQlikSerial(date) {
4740
+ const ms = types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
4741
+ if (Number.isNaN(ms)) {
4742
+ throw new QvdValidationError("dateToQlikSerial needs a valid Date", {
4743
+ provided: describeType(date),
4744
+ type: typeof date
4745
+ });
4746
+ }
4747
+ return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
4748
+ }
4749
+ __name(dateToQlikSerial, "dateToQlikSerial");
4750
+
2877
4751
  // src/index.js
2878
- init_QvdSymbol();
2879
4752
  init_QvdDataFrame();
2880
4753
  init_QvdColumnTable();
2881
4754
  init_QvdFileReader();
2882
4755
  init_QvdFileWriter();
2883
4756
  init_QvdErrors();
2884
4757
 
2885
- export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError };
4758
+ export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdDual, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError, dateToQlikSerial, qlikSerialToDate };
2886
4759
  //# sourceMappingURL=index.js.map
2887
4760
  //# sourceMappingURL=index.js.map