qvdjs 0.10.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -7,6 +7,7 @@ var xml2 = require('xml2js');
7
7
  var assert2 = require('assert');
8
8
  var os = require('os');
9
9
  var v8 = require('v8');
10
+ var util = require('util');
10
11
 
11
12
  function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
12
13
 
@@ -20,6 +21,7 @@ var v8__default = /*#__PURE__*/_interopDefault(v8);
20
21
 
21
22
  var __defProp = Object.defineProperty;
22
23
  var __getOwnPropNames = Object.getOwnPropertyNames;
24
+ var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
23
25
  var __esm = (fn, res) => function __init() {
24
26
  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
25
27
  };
@@ -29,10 +31,14 @@ var __export = (target, all) => {
29
31
  };
30
32
 
31
33
  // src/QvdErrors.js
32
- exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
34
+ var ERROR_NAMES; exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
33
35
  var init_QvdErrors = __esm({
34
36
  "src/QvdErrors.js"() {
37
+ ERROR_NAMES = /* @__PURE__ */ new Map();
35
38
  exports.QvdError = class extends Error {
39
+ static {
40
+ __name(this, "QvdError");
41
+ }
36
42
  /**
37
43
  * Constructs a new QVD error.
38
44
  *
@@ -42,13 +48,16 @@ var init_QvdErrors = __esm({
42
48
  */
43
49
  constructor(message, code, context = {}) {
44
50
  super(message);
45
- this.name = this.constructor.name;
51
+ this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
46
52
  this.code = code;
47
53
  this.context = context;
48
54
  Error.captureStackTrace(this, this.constructor);
49
55
  }
50
56
  };
51
57
  exports.QvdParseError = class extends exports.QvdError {
58
+ static {
59
+ __name(this, "QvdParseError");
60
+ }
52
61
  /**
53
62
  * Constructs a new QVD parse error.
54
63
  *
@@ -60,6 +69,9 @@ var init_QvdErrors = __esm({
60
69
  }
61
70
  };
62
71
  exports.QvdValidationError = class extends exports.QvdError {
72
+ static {
73
+ __name(this, "QvdValidationError");
74
+ }
63
75
  /**
64
76
  * Constructs a new QVD validation error.
65
77
  *
@@ -71,6 +83,9 @@ var init_QvdErrors = __esm({
71
83
  }
72
84
  };
73
85
  exports.QvdIOError = class extends exports.QvdError {
86
+ static {
87
+ __name(this, "QvdIOError");
88
+ }
74
89
  /**
75
90
  * Constructs a new QVD IO error.
76
91
  *
@@ -82,6 +97,9 @@ var init_QvdErrors = __esm({
82
97
  }
83
98
  };
84
99
  exports.QvdCorruptedError = class extends exports.QvdError {
100
+ static {
101
+ __name(this, "QvdCorruptedError");
102
+ }
85
103
  /**
86
104
  * Constructs a new QVD corrupted error.
87
105
  *
@@ -93,6 +111,9 @@ var init_QvdErrors = __esm({
93
111
  }
94
112
  };
95
113
  exports.QvdSecurityError = class extends exports.QvdError {
114
+ static {
115
+ __name(this, "QvdSecurityError");
116
+ }
96
117
  /**
97
118
  * Constructs a new QVD security error.
98
119
  *
@@ -103,163 +124,591 @@ var init_QvdErrors = __esm({
103
124
  super(message, "QVD_SECURITY_ERROR", context);
104
125
  }
105
126
  };
127
+ ERROR_NAMES.set(exports.QvdError, "QvdError");
128
+ ERROR_NAMES.set(exports.QvdParseError, "QvdParseError");
129
+ ERROR_NAMES.set(exports.QvdValidationError, "QvdValidationError");
130
+ ERROR_NAMES.set(exports.QvdIOError, "QvdIOError");
131
+ ERROR_NAMES.set(exports.QvdCorruptedError, "QvdCorruptedError");
132
+ ERROR_NAMES.set(exports.QvdSecurityError, "QvdSecurityError");
106
133
  }
107
134
  });
108
135
 
109
- // src/QvdSymbol.js
110
- exports.QvdSymbol = void 0;
111
- var init_QvdSymbol = __esm({
112
- "src/QvdSymbol.js"() {
136
+ // src/util/cellRules.js
137
+ function asDual(value) {
138
+ if (value === null || typeof value !== "object") {
139
+ return null;
140
+ }
141
+ try {
142
+ if (value[DUAL_BRAND] === true) {
143
+ return value;
144
+ }
145
+ if (!isPlainObject(value)) {
146
+ return null;
147
+ }
148
+ const keys = Object.keys(value);
149
+ return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
150
+ } catch {
151
+ return null;
152
+ }
153
+ }
154
+ function isPlainObject(value) {
155
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
156
+ return false;
157
+ }
158
+ const prototype = Object.getPrototypeOf(value);
159
+ return prototype === null || Object.getPrototypeOf(prototype) === null;
160
+ }
161
+ function isNumericText(text) {
162
+ return text.trim() !== "" && Number.isFinite(Number(text));
163
+ }
164
+ function isStoredAsInt(value) {
165
+ return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
166
+ }
167
+ function numberProblem(value) {
168
+ if (typeof value !== "number") {
169
+ return `is ${describeType(value)}, not a number`;
170
+ }
171
+ return Number.isFinite(value) ? null : `is ${String(value)}`;
172
+ }
173
+ function textProblem(value) {
174
+ if (typeof value !== "string") {
175
+ return { reason: "type" };
176
+ }
177
+ const position = value.indexOf(NUL);
178
+ if (position !== -1) {
179
+ return { reason: "nul", position };
180
+ }
181
+ return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
182
+ }
183
+ function unpairedSurrogateIndex(value) {
184
+ for (let index = 0; index < value.length; index++) {
185
+ const unit = value.charCodeAt(index);
186
+ if (unit < 55296 || unit > 57343) {
187
+ continue;
188
+ }
189
+ const next = value.charCodeAt(index + 1);
190
+ if (unit <= 56319 && next >= 56320 && next <= 57343) {
191
+ index++;
192
+ continue;
193
+ }
194
+ return index;
195
+ }
196
+ return -1;
197
+ }
198
+ function checkNumber(value, subject, context) {
199
+ if (numberProblem(value) === null) {
200
+ return;
201
+ }
202
+ if (subject === null && typeof value === "number") {
203
+ throw new exports.QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
204
+ ...context,
205
+ provided: String(value)
206
+ });
207
+ }
208
+ throw new exports.QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
209
+ ...context,
210
+ type: typeof value
211
+ });
212
+ }
213
+ function checkText(value, subject, context) {
214
+ const problem = textProblem(value);
215
+ if (problem === null) {
216
+ return;
217
+ }
218
+ if (problem.reason === "type") {
219
+ throw new exports.QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
220
+ ...context,
221
+ type: typeof value
222
+ });
223
+ }
224
+ if (problem.reason === "surrogate") {
225
+ throw new exports.QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
226
+ ...context,
227
+ position: problem.position
228
+ });
229
+ }
230
+ throw new exports.QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
231
+ }
232
+ function constructorName(value) {
233
+ try {
234
+ const name = value.constructor?.name;
235
+ return typeof name === "string" && name !== "" ? name : null;
236
+ } catch {
237
+ return null;
238
+ }
239
+ }
240
+ function describeType(value) {
241
+ if (value === null) {
242
+ return "null";
243
+ }
244
+ if (typeof value === "number") {
245
+ return Number.isFinite(value) ? "a number" : String(value);
246
+ }
247
+ if (typeof value === "undefined") {
248
+ return "undefined";
249
+ }
250
+ if (typeof value !== "object") {
251
+ return `a ${typeof value}`;
252
+ }
253
+ let name;
254
+ try {
255
+ if (Array.isArray(value)) {
256
+ return "an array";
257
+ }
258
+ name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
259
+ if (name === "Object") {
260
+ if (!isPlainObject(value)) {
261
+ return "an object whose prototype is not Object.prototype";
262
+ }
263
+ const keys = Object.keys(value);
264
+ if (keys.length === 0) {
265
+ return "a plain object";
266
+ }
267
+ return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
268
+ }
269
+ } catch {
270
+ return "an object";
271
+ }
272
+ return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
273
+ }
274
+ var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
275
+ var init_cellRules = __esm({
276
+ "src/util/cellRules.js"() {
113
277
  init_QvdErrors();
114
- exports.QvdSymbol = class _QvdSymbol {
115
- /**
116
- * Constructs a new QVD symbol.
117
- *
118
- * @param {number|null} intValue The integer value.
119
- * @param {number|null} doubleValue The double value.
120
- * @param {string|null} stringValue The string value.
121
- */
122
- constructor(intValue, doubleValue, stringValue) {
123
- this._intValue = intValue;
124
- this._doubleValue = doubleValue;
125
- this._stringValue = stringValue;
278
+ INT32_MIN = -2147483648;
279
+ INT32_MAX = 2147483647;
280
+ NUL = String.fromCharCode(0);
281
+ DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
282
+ __name(asDual, "asDual");
283
+ __name(isPlainObject, "isPlainObject");
284
+ __name(isNumericText, "isNumericText");
285
+ __name(isStoredAsInt, "isStoredAsInt");
286
+ __name(numberProblem, "numberProblem");
287
+ __name(textProblem, "textProblem");
288
+ __name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
289
+ __name(checkNumber, "checkNumber");
290
+ __name(checkText, "checkText");
291
+ __name(constructorName, "constructorName");
292
+ __name(describeType, "describeType");
293
+ }
294
+ });
295
+
296
+ // src/util/symbolBytes.js
297
+ function kindOf(number, text) {
298
+ if (number === null) {
299
+ return 4;
300
+ }
301
+ if (isStoredAsInt(number)) {
302
+ return text === null ? 1 : 5;
303
+ }
304
+ return text === null ? 2 : 6;
305
+ }
306
+ function symbolByteLength(kind, number, text) {
307
+ const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
308
+ const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
309
+ return 1 + numberBytes + textBytes;
310
+ }
311
+ function writeSymbol(buffer, offset, kind, number, text) {
312
+ buffer[offset++] = kind;
313
+ if (kind === 1 || kind === 5) {
314
+ offset = buffer.writeInt32LE(number, offset);
315
+ } else if (kind === 2 || kind === 6) {
316
+ offset = buffer.writeDoubleLE(number, offset);
317
+ }
318
+ if (kind >= 4) {
319
+ offset += buffer.write(text, offset, "utf8");
320
+ buffer[offset++] = 0;
321
+ }
322
+ return offset;
323
+ }
324
+ var init_symbolBytes = __esm({
325
+ "src/util/symbolBytes.js"() {
326
+ init_cellRules();
327
+ __name(kindOf, "kindOf");
328
+ __name(symbolByteLength, "symbolByteLength");
329
+ __name(writeSymbol, "writeSymbol");
330
+ }
331
+ });
332
+
333
+ // src/QvdDual.js
334
+ function defineHalves(target, number, text) {
335
+ Object.defineProperties(target, {
336
+ number: { value: number, enumerable: true },
337
+ text: { value: text, enumerable: true }
338
+ });
339
+ Object.freeze(target);
340
+ }
341
+ function dualFromSymbol(number, text) {
342
+ const dual = Object.create(exports.QvdDual.prototype);
343
+ defineHalves(dual, number, text);
344
+ return dual;
345
+ }
346
+ exports.QvdDual = void 0;
347
+ var init_QvdDual = __esm({
348
+ "src/QvdDual.js"() {
349
+ init_cellRules();
350
+ exports.QvdDual = class {
351
+ static {
352
+ __name(this, "QvdDual");
126
353
  }
127
354
  /**
128
- * Returns the integer value of this symbol.
355
+ * Constructs a dual value.
356
+ *
357
+ * The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
358
+ * integer inside the int32 range is stored as a dual int, anything else as a dual double.
129
359
  *
130
- * @return {number|null} The integer value.
360
+ * @param {number} number The numeric half. Must be a finite number.
361
+ * @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
362
+ * @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
131
363
  */
132
- get intValue() {
133
- return this._intValue;
364
+ constructor(number, text) {
365
+ checkNumber(number, "The number of a dual value", { half: "number" });
366
+ checkText(text, "The text of a dual value", { half: "text" });
367
+ defineHalves(this, number, text);
134
368
  }
135
369
  /**
136
- * Returns the double value of this symbol.
370
+ * Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
137
371
  *
138
- * @return {number|null} The double value.
372
+ * The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
373
+ * conversion ran, not which half the caller meant, and the fix is the same for all three.
374
+ *
375
+ * @return {never}
376
+ * @throws {TypeError} Always.
139
377
  */
140
- get doubleValue() {
141
- return this._doubleValue;
378
+ [Symbol.toPrimitive]() {
379
+ throw new TypeError(
380
+ `A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
381
+ );
142
382
  }
143
383
  /**
144
- * Returns the string value of this symbol.
384
+ * Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
145
385
  *
146
- * @return {string|null} The string value.
386
+ * @return {{number: number, text: string}} The dual as a plain object.
147
387
  */
148
- get stringValue() {
149
- return this._stringValue;
388
+ toJSON() {
389
+ return { number: this.number, text: this.text };
150
390
  }
151
391
  /**
152
- * Retrieves the primary value of this symbol. The primary value is descriptive raw value.
153
- * It is either the string value, the integer value or the double value, prioritized in this order.
392
+ * How Node's `util.inspect` and `console.log` show a dual.
154
393
  *
155
- * @return {number|string|null} The primary value.
394
+ * @return {string} For example `QvdDual(4.5, "4.50")`.
156
395
  */
157
- toPrimaryValue() {
158
- if (null != this._stringValue) {
159
- return this._stringValue;
160
- } else if (null != this._intValue) {
161
- return this._intValue;
162
- } else if (null != this._doubleValue) {
163
- return this._doubleValue;
164
- } else {
165
- return null;
166
- }
396
+ [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
397
+ return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
167
398
  }
168
- /**
169
- * Converts the symbol to its byte representation.
170
- *
171
- * @return {Buffer} The byte representation of the symbol.
172
- */
173
- toByteRepresentation() {
174
- if (this._intValue !== null && this._stringValue !== null) {
175
- const intBuffer = Buffer.alloc(4);
176
- intBuffer.writeInt32LE(this._intValue);
177
- const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
178
- return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
179
- } else if (this._doubleValue !== null && this._stringValue !== null) {
180
- const floatBuffer = Buffer.alloc(8);
181
- floatBuffer.writeDoubleLE(this._doubleValue);
182
- const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
183
- return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
184
- } else if (this._intValue !== null) {
185
- const buffer = Buffer.alloc(4);
186
- buffer.writeInt32LE(this._intValue);
187
- return Buffer.concat([Buffer.from([1]), buffer]);
188
- } else if (this._doubleValue !== null) {
189
- const buffer = Buffer.alloc(8);
190
- buffer.writeDoubleLE(this._doubleValue);
191
- return Buffer.concat([Buffer.from([2]), buffer]);
192
- } else if (this._stringValue !== null) {
193
- const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
194
- return Buffer.concat([Buffer.from([4]), buffer]);
195
- } else {
196
- throw new exports.QvdValidationError("The symbol does not contain any value.", {
197
- intValue: this._intValue,
198
- doubleValue: this._doubleValue,
199
- stringValue: this._stringValue
200
- });
201
- }
399
+ /** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
400
+ get [Symbol.toStringTag]() {
401
+ return "QvdDual";
202
402
  }
203
403
  /**
204
- * Checks if this symbol is equal to another symbol.
404
+ * Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
405
+ * own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
205
406
  *
206
- * @param {*} value The object to compare with.
207
- * @return {boolean} True if the objects are equal, false otherwise.
208
- */
209
- equals(value) {
210
- if (!(value instanceof _QvdSymbol)) {
211
- return false;
212
- }
213
- return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
214
- }
215
- /**
216
- * Constructs a pure integer value symbol.
407
+ * Recognition only. The halves are checked when the value is written.
217
408
  *
218
- * @param {number} intValue The integer value.
219
- * @return {QvdSymbol} The constructed value symbol.
409
+ * @param {any} value Any value.
410
+ * @return {boolean} True for a dual cell.
220
411
  */
221
- static fromIntValue(intValue) {
222
- return new _QvdSymbol(intValue, null, null);
412
+ static isDual(value) {
413
+ return asDual(value) !== null;
223
414
  }
224
- /**
225
- * Constructs a pure double value symbol.
226
- *
227
- * @param {number} doubleValue The double value.
228
- * @return {QvdSymbol} The constructed value symbol.
229
- */
230
- static fromDoubleValue(doubleValue) {
231
- return new _QvdSymbol(null, doubleValue, null);
415
+ };
416
+ Object.defineProperty(exports.QvdDual.prototype, DUAL_BRAND, { value: true });
417
+ __name(defineHalves, "defineHalves");
418
+ __name(dualFromSymbol, "dualFromSymbol");
419
+ }
420
+ });
421
+
422
+ // src/util/readOptions.js
423
+ function requireRowCount(value, name, filePath) {
424
+ if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
425
+ throw new exports.QvdValidationError(`${name} must be a non-negative integer`, {
426
+ option: name,
427
+ provided: value,
428
+ type: typeof value,
429
+ file: filePath
430
+ });
431
+ }
432
+ return value;
433
+ }
434
+ function normaliseWindow(window, filePath) {
435
+ if (window === null || window === void 0) {
436
+ return { offset: 0, limit: null };
437
+ }
438
+ if (typeof window === "number") {
439
+ return { offset: 0, limit: requireRowCount(window, "maxRows", filePath) };
440
+ }
441
+ if (typeof window !== "object" || Array.isArray(window)) {
442
+ throw new exports.QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
443
+ provided: window,
444
+ type: typeof window,
445
+ file: filePath
446
+ });
447
+ }
448
+ const { offset, limit, maxRows } = window;
449
+ const limitGiven = limit !== void 0 && limit !== null;
450
+ const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
451
+ if (limitGiven && maxRowsGiven) {
452
+ throw new exports.QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
453
+ maxRows,
454
+ limit,
455
+ file: filePath
456
+ });
457
+ }
458
+ return {
459
+ offset: offset === void 0 || offset === null ? 0 : requireRowCount(offset, "offset", filePath),
460
+ limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
461
+ };
462
+ }
463
+ function resolveWindow(window, totalRows) {
464
+ const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
465
+ const offset = Math.min(window.offset, rows);
466
+ return {
467
+ offset,
468
+ limit: Math.max(0, Math.min(window.limit === null ? Infinity : window.limit, rows - offset))
469
+ };
470
+ }
471
+ function selectFields(fields, requested, filePath) {
472
+ if (requested === null || requested === void 0) {
473
+ return fields;
474
+ }
475
+ if (!Array.isArray(requested)) {
476
+ throw new exports.QvdValidationError("fields must be an array of field names", {
477
+ provided: requested,
478
+ type: typeof requested,
479
+ file: filePath
480
+ });
481
+ }
482
+ const available = fields.map((field) => field["FieldName"]);
483
+ if (requested.length === 0) {
484
+ throw new exports.QvdValidationError("fields must name at least one field", {
485
+ availableColumns: available,
486
+ file: filePath
487
+ });
488
+ }
489
+ const seen = /* @__PURE__ */ new Set();
490
+ return requested.map((name) => {
491
+ if (typeof name !== "string") {
492
+ throw new exports.QvdValidationError("Field names must be strings", {
493
+ provided: name,
494
+ type: typeof name,
495
+ availableColumns: available,
496
+ file: filePath
497
+ });
498
+ }
499
+ if (seen.has(name)) {
500
+ throw new exports.QvdValidationError(`Field '${name}' is listed twice`, {
501
+ column: name,
502
+ fields: requested,
503
+ file: filePath
504
+ });
505
+ }
506
+ seen.add(name);
507
+ const index = available.indexOf(name);
508
+ if (index === -1) {
509
+ throw new exports.QvdValidationError(`Column '${name}' does not exist`, {
510
+ column: name,
511
+ availableColumns: available,
512
+ file: filePath
513
+ });
514
+ }
515
+ return fields[index];
516
+ });
517
+ }
518
+ function normaliseDuals(value, filePath) {
519
+ if (value === void 0 || value === null) {
520
+ return "number";
521
+ }
522
+ if (!DUAL_MODES.includes(value)) {
523
+ throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
524
+ option: "duals",
525
+ provided: value,
526
+ file: filePath
527
+ });
528
+ }
529
+ return value;
530
+ }
531
+ function normaliseCoerceNumericStrings(value, filePath) {
532
+ if (value === void 0 || value === null) {
533
+ return false;
534
+ }
535
+ if (typeof value !== "boolean") {
536
+ throw new exports.QvdValidationError("coerceNumericStrings must be true or false", {
537
+ option: "coerceNumericStrings",
538
+ provided: value,
539
+ type: typeof value,
540
+ file: filePath
541
+ });
542
+ }
543
+ return value;
544
+ }
545
+ function readerOptionsFrom(options) {
546
+ return {
547
+ allowedDir: options.allowedDir,
548
+ memorySafetyFactor: options.memorySafetyFactor,
549
+ symbolFilteringThreshold: options.symbolFilteringThreshold,
550
+ fields: options.fields === void 0 ? null : options.fields,
551
+ duals: options.duals,
552
+ coerceNumericStrings: options.coerceNumericStrings,
553
+ onProgress: options.onProgress,
554
+ signal: options.signal
555
+ };
556
+ }
557
+ function metadataOptionsFrom(options) {
558
+ return {
559
+ allowedDir: options.allowedDir,
560
+ onProgress: options.onProgress,
561
+ signal: options.signal
562
+ };
563
+ }
564
+ function windowFrom(options) {
565
+ return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
566
+ }
567
+ var DUAL_MODES;
568
+ var init_readOptions = __esm({
569
+ "src/util/readOptions.js"() {
570
+ init_QvdErrors();
571
+ __name(requireRowCount, "requireRowCount");
572
+ __name(normaliseWindow, "normaliseWindow");
573
+ __name(resolveWindow, "resolveWindow");
574
+ __name(selectFields, "selectFields");
575
+ DUAL_MODES = Object.freeze(["number", "text", "both"]);
576
+ __name(normaliseDuals, "normaliseDuals");
577
+ __name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
578
+ __name(readerOptionsFrom, "readerOptionsFrom");
579
+ __name(metadataOptionsFrom, "metadataOptionsFrom");
580
+ __name(windowFrom, "windowFrom");
581
+ }
582
+ });
583
+
584
+ // src/util/storedSymbols.js
585
+ function trustStoredSymbols(entries) {
586
+ const record = Object.freeze(entries);
587
+ trusted.add(record);
588
+ return record;
589
+ }
590
+ function attachStoredSymbols(metadata, record) {
591
+ if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
592
+ Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
593
+ }
594
+ }
595
+ function refuse(message, context) {
596
+ throw new exports.QvdValidationError(message, context);
597
+ }
598
+ function normaliseStoredSymbols(record) {
599
+ if (record === null || record === void 0) {
600
+ return null;
601
+ }
602
+ if (typeof record === "object" && trusted.has(record)) {
603
+ return record;
604
+ }
605
+ if (!Array.isArray(record)) {
606
+ refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
607
+ option: "storedSymbols",
608
+ type: typeof record
609
+ });
610
+ }
611
+ const fields = /* @__PURE__ */ new Set();
612
+ const entries = record.map((entry, index) => {
613
+ if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
614
+ refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
615
+ }
616
+ const { field, values, numbers, texts } = entry;
617
+ if (fields.has(field)) {
618
+ refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
619
+ }
620
+ fields.add(field);
621
+ if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
622
+ refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
623
+ field,
624
+ entry: index
625
+ });
626
+ }
627
+ for (let symbol = 0; symbol < values.length; symbol++) {
628
+ const value = values[symbol];
629
+ const number = numbers[symbol];
630
+ const text = texts[symbol];
631
+ const context = { field, symbol };
632
+ if (typeof value !== "number" && typeof value !== "string") {
633
+ refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
634
+ ...context,
635
+ type: typeof value
636
+ });
232
637
  }
233
- /**
234
- * Constructs a pure string value symbol.
235
- *
236
- * @param {string} stringValue The string value.
237
- * @return {QvdSymbol} The constructed value symbol.
238
- */
239
- static fromStringValue(stringValue) {
240
- return new _QvdSymbol(null, null, stringValue);
638
+ if (number !== null) {
639
+ checkNumber(number, "A stored symbol's number", context);
241
640
  }
242
- /**
243
- * Constructs a dual value symbol from an integer and a string value.
244
- *
245
- * @param {number} intValue The integer value.
246
- * @param {string} stringValue The string value.
247
- * @return {QvdSymbol} The constructed value symbol.
248
- */
249
- static fromDualIntValue(intValue, stringValue) {
250
- return new _QvdSymbol(intValue, null, stringValue);
641
+ if (text !== null) {
642
+ checkText(text, "A stored symbol's text", context);
251
643
  }
252
- /**
253
- * Constructs a dual value symbol from a double and a string value.
254
- *
255
- * @param {number} doubleValue The double value.
256
- * @param {string} stringValue The string value.
257
- * @return {QvdSymbol} The constructed value symbol.
258
- */
259
- static fromDualDoubleValue(doubleValue, stringValue) {
260
- return new _QvdSymbol(null, doubleValue, stringValue);
644
+ const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
645
+ if (!consistent) {
646
+ refuse(
647
+ number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
648
+ { ...context, value, number, text }
649
+ );
261
650
  }
262
- };
651
+ }
652
+ return Object.freeze({
653
+ field,
654
+ values: Object.freeze(values.slice()),
655
+ numbers: Object.freeze(numbers.slice()),
656
+ texts: Object.freeze(texts.slice())
657
+ });
658
+ });
659
+ return trustStoredSymbols(entries);
660
+ }
661
+ function narrowStoredSymbols(record, columns) {
662
+ if (record === null) {
663
+ return null;
664
+ }
665
+ const kept = record.filter((entry) => columns.includes(entry.field));
666
+ return kept.length === record.length ? record : trustStoredSymbols(kept);
667
+ }
668
+ function storedSymbolsEntry(record, field) {
669
+ if (record === null) {
670
+ return null;
671
+ }
672
+ return record.find((entry) => entry.field === field) ?? null;
673
+ }
674
+ function firstTextByValue(entry) {
675
+ const byValue = /* @__PURE__ */ new Map();
676
+ for (let index = entry.values.length - 1; index >= 0; index--) {
677
+ const text = entry.texts[index];
678
+ if (text !== null) {
679
+ byValue.set(entry.values[index], text);
680
+ }
681
+ }
682
+ return byValue;
683
+ }
684
+ function storedTextOf(entry, value) {
685
+ let byValue = firstTexts.get(entry);
686
+ if (byValue === void 0) {
687
+ byValue = firstTextByValue(entry);
688
+ firstTexts.set(entry, byValue);
689
+ }
690
+ return byValue.get(value) ?? null;
691
+ }
692
+ function sameValueZero(a, b) {
693
+ return a === b || a !== a && b !== b;
694
+ }
695
+ var STORED_SYMBOLS, trusted, firstTexts;
696
+ var init_storedSymbols = __esm({
697
+ "src/util/storedSymbols.js"() {
698
+ init_QvdErrors();
699
+ init_cellRules();
700
+ STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
701
+ trusted = /* @__PURE__ */ new WeakSet();
702
+ __name(trustStoredSymbols, "trustStoredSymbols");
703
+ __name(attachStoredSymbols, "attachStoredSymbols");
704
+ __name(refuse, "refuse");
705
+ __name(normaliseStoredSymbols, "normaliseStoredSymbols");
706
+ __name(narrowStoredSymbols, "narrowStoredSymbols");
707
+ __name(storedSymbolsEntry, "storedSymbolsEntry");
708
+ firstTexts = /* @__PURE__ */ new WeakMap();
709
+ __name(firstTextByValue, "firstTextByValue");
710
+ __name(storedTextOf, "storedTextOf");
711
+ __name(sameValueZero, "sameValueZero");
263
712
  }
264
713
  });
265
714
  function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
@@ -364,6 +813,10 @@ function validatePath(filePath, allowedDir) {
364
813
  var init_validatePath = __esm({
365
814
  "src/util/validatePath.js"() {
366
815
  init_QvdErrors();
816
+ __name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
817
+ __name(resolveDeepestExisting, "resolveDeepestExisting");
818
+ __name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
819
+ __name(validatePath, "validatePath");
367
820
  }
368
821
  });
369
822
 
@@ -413,6 +866,9 @@ var init_bitUtils = __esm({
413
866
  "src/util/bitUtils.js"() {
414
867
  MAX_BIT_WIDTH = 31;
415
868
  POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
869
+ __name(fieldGeometry, "fieldGeometry");
870
+ __name(decodeIndexColumn, "decodeIndexColumn");
871
+ __name(writeBitField, "writeBitField");
416
872
  }
417
873
  });
418
874
 
@@ -421,14 +877,310 @@ var QvdFileWriter_exports = {};
421
877
  __export(QvdFileWriter_exports, {
422
878
  QvdFileWriter: () => exports.QvdFileWriter
423
879
  });
424
- exports.QvdFileWriter = void 0;
880
+ function notXmlIndex(value) {
881
+ for (let index = 0; index < value.length; index++) {
882
+ const unit = value.charCodeAt(index);
883
+ if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
884
+ return index;
885
+ }
886
+ }
887
+ return -1;
888
+ }
889
+ function checkHeaderText(value, subject, context) {
890
+ checkText(value, subject, context);
891
+ const position = notXmlIndex(value);
892
+ if (position !== -1) {
893
+ const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
894
+ throw new exports.QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
895
+ }
896
+ }
897
+ function checkHeaderTexts(value, property, owner, context) {
898
+ if (typeof value === "string") {
899
+ checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
900
+ } else if (Array.isArray(value)) {
901
+ value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
902
+ } else if (value !== null && typeof value === "object") {
903
+ for (const [key, item] of Object.entries(value)) {
904
+ checkHeaderTexts(item, `${property}.${key}`, owner, context);
905
+ }
906
+ }
907
+ }
908
+ function validateColumnNames(columns, filePath) {
909
+ const seen = /* @__PURE__ */ new Set();
910
+ columns.forEach((name, index) => {
911
+ if (typeof name !== "string" || name.length === 0) {
912
+ throw new exports.QvdValidationError("Field names must be non-empty strings", {
913
+ column: index,
914
+ provided: name,
915
+ type: typeof name,
916
+ file: filePath,
917
+ stage: "buildSymbolTable"
918
+ });
919
+ }
920
+ checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
921
+ if (seen.has(name)) {
922
+ throw new exports.QvdValidationError(`Field '${name}' appears twice`, {
923
+ column: name,
924
+ columnIndex: index,
925
+ file: filePath,
926
+ stage: "buildSymbolTable"
927
+ });
928
+ }
929
+ seen.add(name);
930
+ });
931
+ }
932
+ function refuseCell(value, column, row, filePath) {
933
+ let resemblesDual = false;
934
+ try {
935
+ resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
936
+ } catch {
937
+ }
938
+ throw new exports.QvdValidationError(
939
+ `A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
940
+ {
941
+ column,
942
+ row,
943
+ type: typeof value,
944
+ constructor: constructorName(value) ?? void 0,
945
+ file: filePath,
946
+ stage: "buildSymbolTable"
947
+ }
948
+ );
949
+ }
950
+ function valuesAreNumbers(entry) {
951
+ const { values, numbers } = entry;
952
+ for (let index = 0; index < values.length; index++) {
953
+ if (!sameValueZero(values[index], numbers[index])) {
954
+ return false;
955
+ }
956
+ }
957
+ return true;
958
+ }
959
+ function newSlot(column, key, text) {
960
+ const slot = column.keys.length;
961
+ column.keys.push(key);
962
+ column.texts.push(text);
963
+ if (typeof key === "number") {
964
+ column.facts.hasNumber = true;
965
+ if (!Number.isInteger(key)) column.facts.hasFraction = true;
966
+ } else {
967
+ column.facts.hasNonNumber = true;
968
+ }
969
+ return slot;
970
+ }
971
+ function slotFor(column, key, text) {
972
+ let slot = column.byKey.get(key);
973
+ if (slot === void 0) {
974
+ slot = newSlot(column, key, text);
975
+ column.byKey.set(key, slot);
976
+ } else if (text !== null && column.texts[slot] === null && typeof key === "number") {
977
+ column.texts[slot] = text;
978
+ }
979
+ return slot;
980
+ }
981
+ function standsForSeveral(indices, numbers, texts) {
982
+ let number = null;
983
+ let string = null;
984
+ for (const index of indices) {
985
+ if (numbers[index] !== null) {
986
+ if (number === null) {
987
+ number = numbers[index];
988
+ } else if (!sameValueZero(number, numbers[index])) {
989
+ return true;
990
+ }
991
+ } else if (string === null) {
992
+ string = texts[index];
993
+ } else if (string !== texts[index]) {
994
+ return true;
995
+ }
996
+ }
997
+ return number !== null && string !== null;
998
+ }
999
+ function ambiguousAsUncoercedText(entry) {
1000
+ const { values, numbers, texts } = entry;
1001
+ for (let index = 0; index < values.length; index++) {
1002
+ if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
1003
+ if (!isNumericText(texts[index])) {
1004
+ return false;
1005
+ }
1006
+ }
1007
+ }
1008
+ const byShown = /* @__PURE__ */ new Map();
1009
+ for (let index = 0; index < values.length; index++) {
1010
+ const shown = texts[index] ?? numbers[index];
1011
+ const indices = byShown.get(shown);
1012
+ if (indices === void 0) {
1013
+ byShown.set(shown, [index]);
1014
+ } else {
1015
+ indices.push(index);
1016
+ }
1017
+ }
1018
+ for (const indices of byShown.values()) {
1019
+ if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
1020
+ return true;
1021
+ }
1022
+ }
1023
+ return false;
1024
+ }
1025
+ function unambiguousWithoutDuals(column) {
1026
+ const { numbers, texts } = column.entry;
1027
+ for (const found of column.byValue.values()) {
1028
+ if (typeof found !== "number") {
1029
+ const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
1030
+ if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
1031
+ return false;
1032
+ }
1033
+ }
1034
+ }
1035
+ return true;
1036
+ }
1037
+ function ambiguityRemedy(column) {
1038
+ const { values, numbers, texts } = column.entry;
1039
+ let coerced = false;
1040
+ let other = false;
1041
+ for (const [value, found] of column.byValue) {
1042
+ if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
1043
+ if (typeof value === "number") {
1044
+ coerced = true;
1045
+ } else {
1046
+ other = true;
1047
+ }
1048
+ }
1049
+ }
1050
+ if (!coerced) {
1051
+ return "Read the field with {duals: 'both'}, or write QvdDual cells.";
1052
+ }
1053
+ if (!other && !ambiguousAsUncoercedText(column.entry)) {
1054
+ return "Read the field without {coerceNumericStrings: true}.";
1055
+ }
1056
+ if (unambiguousWithoutDuals(column)) {
1057
+ return "Read the field with {duals: 'both'}.";
1058
+ }
1059
+ return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
1060
+ }
1061
+ function slotForCell(column, value, row, filePath) {
1062
+ if (column.byCell === column.byKey) {
1063
+ return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
1064
+ }
1065
+ const found = column.byValue.get(value);
1066
+ if (found === void 0) {
1067
+ return slotFor(column, value, null);
1068
+ }
1069
+ const { numbers, texts } = column.entry;
1070
+ const indices = typeof found === "number" ? [found] : found;
1071
+ if (standsForSeveral(indices, numbers, texts)) {
1072
+ const stored = [];
1073
+ for (const index of indices) {
1074
+ if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
1075
+ stored.push({ number: numbers[index], text: texts[index] });
1076
+ }
1077
+ }
1078
+ throw new exports.QvdValidationError(
1079
+ `The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
1080
+ { column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
1081
+ );
1082
+ }
1083
+ let number = null;
1084
+ let numberText = null;
1085
+ for (const index of indices) {
1086
+ if (numbers[index] === null) {
1087
+ return slotFor(column, texts[index], null);
1088
+ }
1089
+ number ??= numbers[index];
1090
+ numberText ??= texts[index];
1091
+ }
1092
+ return slotFor(column, number, numberText);
1093
+ }
1094
+ function slotForObject(column, value, row, filePath) {
1095
+ const dual = asDual(value);
1096
+ if (dual === null) {
1097
+ refuseCell(value, column.name, row, filePath);
1098
+ }
1099
+ const { number, text } = dual;
1100
+ if (numberProblem(number) !== null) {
1101
+ checkNumber(number, "The number of a dual value", {
1102
+ column: column.name,
1103
+ row,
1104
+ half: "number",
1105
+ file: filePath,
1106
+ stage: "buildSymbolTable"
1107
+ });
1108
+ }
1109
+ if (textProblem(text) !== null) {
1110
+ checkText(text, "The text of a dual value", {
1111
+ column: column.name,
1112
+ row,
1113
+ half: "text",
1114
+ file: filePath,
1115
+ stage: "buildSymbolTable"
1116
+ });
1117
+ }
1118
+ const slot = slotFor(column, number, text);
1119
+ if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
1120
+ column.byObject.set(value, slot);
1121
+ }
1122
+ return slot;
1123
+ }
1124
+ function pruneContradictedTags(tags, facts) {
1125
+ if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
1126
+ return tags || {};
1127
+ }
1128
+ const list = Array.isArray(tags.String) ? tags.String : [tags.String];
1129
+ const kept = list.filter((tag) => {
1130
+ if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
1131
+ if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
1132
+ if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
1133
+ return true;
1134
+ });
1135
+ if (kept.length === list.length) {
1136
+ return tags;
1137
+ }
1138
+ return kept.length === 0 ? {} : { ...tags, String: kept };
1139
+ }
1140
+ function resetContradictedNumberFormat(numberFormat, facts) {
1141
+ if (!numberFormat) {
1142
+ return { ...UNKNOWN_NUMBER_FORMAT };
1143
+ }
1144
+ if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
1145
+ return { ...UNKNOWN_NUMBER_FORMAT };
1146
+ }
1147
+ return numberFormat;
1148
+ }
1149
+ var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT; exports.QvdFileWriter = void 0;
425
1150
  var init_QvdFileWriter = __esm({
426
1151
  "src/QvdFileWriter.js"() {
427
- init_QvdSymbol();
428
1152
  init_QvdErrors();
429
1153
  init_validatePath();
430
1154
  init_bitUtils();
431
- exports.QvdFileWriter = class _QvdFileWriter {
1155
+ init_cellRules();
1156
+ init_symbolBytes();
1157
+ init_storedSymbols();
1158
+ __name(notXmlIndex, "notXmlIndex");
1159
+ __name(checkHeaderText, "checkHeaderText");
1160
+ __name(checkHeaderTexts, "checkHeaderTexts");
1161
+ __name(validateColumnNames, "validateColumnNames");
1162
+ __name(refuseCell, "refuseCell");
1163
+ OBJECT_MEMO_BASE = 64;
1164
+ __name(valuesAreNumbers, "valuesAreNumbers");
1165
+ __name(newSlot, "newSlot");
1166
+ __name(slotFor, "slotFor");
1167
+ __name(standsForSeveral, "standsForSeveral");
1168
+ __name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
1169
+ __name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
1170
+ __name(ambiguityRemedy, "ambiguityRemedy");
1171
+ __name(slotForCell, "slotForCell");
1172
+ __name(slotForObject, "slotForObject");
1173
+ NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
1174
+ TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
1175
+ WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
1176
+ NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
1177
+ UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
1178
+ __name(pruneContradictedTags, "pruneContradictedTags");
1179
+ __name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
1180
+ exports.QvdFileWriter = class {
1181
+ static {
1182
+ __name(this, "QvdFileWriter");
1183
+ }
432
1184
  /**
433
1185
  * Constructs a new QVD file writer.
434
1186
  *
@@ -449,7 +1201,8 @@ var init_QvdFileWriter = __esm({
449
1201
  this._onProgress = onProgress;
450
1202
  this._header = null;
451
1203
  this._symbolBuffer = null;
452
- this._symbolTable = null;
1204
+ this._symbolCounts = null;
1205
+ this._symbolFacts = null;
453
1206
  this._symbolTableMetadata = null;
454
1207
  this._indexBuffer = null;
455
1208
  this._symbolIndexByValue = null;
@@ -560,19 +1313,12 @@ var init_QvdFileWriter = __esm({
560
1313
  BitOffset: this._indexTableMetadata?.[index][0],
561
1314
  BitWidth: this._indexTableMetadata?.[index][1],
562
1315
  Bias: this._indexTableMetadata?.[index][2],
563
- NoOfSymbols: this._symbolTable?.[index].length,
1316
+ NoOfSymbols: this._symbolCounts?.[index],
564
1317
  Offset: this._symbolTableMetadata?.[index][0],
565
1318
  Length: this._symbolTableMetadata?.[index][1],
566
1319
  Comment: existingField?.Comment || "",
567
- NumberFormat: existingField?.NumberFormat || {
568
- Type: "UNKNOWN",
569
- nDec: "0",
570
- UseThou: "0",
571
- Fmt: "",
572
- Dec: "",
573
- Thou: ""
574
- },
575
- Tags: existingField?.Tags || {}
1320
+ NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
1321
+ Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
576
1322
  };
577
1323
  })
578
1324
  },
@@ -582,6 +1328,19 @@ var init_QvdFileWriter = __esm({
582
1328
  Length: this._indexBuffer?.length
583
1329
  }
584
1330
  };
1331
+ const { Fields, ...table } = xmlObject.QvdTableHeader;
1332
+ for (const [property, value] of Object.entries(table)) {
1333
+ checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
1334
+ }
1335
+ for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
1336
+ for (const [property, value] of Object.entries(field)) {
1337
+ checkHeaderTexts(value, property, `field '${FieldName}'`, {
1338
+ column: FieldName,
1339
+ file: this._path,
1340
+ stage: "buildHeader"
1341
+ });
1342
+ }
1343
+ }
585
1344
  const builder = new xml2__default.default.Builder({
586
1345
  renderOpts: {
587
1346
  pretty: true,
@@ -595,25 +1354,49 @@ var init_QvdFileWriter = __esm({
595
1354
  /**
596
1355
  * Builds the symbol table of the QVD file.
597
1356
  *
598
- * PERFORMANCE OPTIMIZATION: This method uses a single-pass algorithm to build
599
- * symbol tables for all columns simultaneously. This reduces time complexity from
600
- * O(n×m×s) to O(n×m) where n=rows, m=columns, s=symbols per column.
1357
+ * One pass over the rows finds each column's distinct values in the order they first appear,
1358
+ * which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
1359
+ * column encodes them: every symbol is sized first, then written into one buffer of exactly that
1360
+ * size.
1361
+ *
1362
+ * What each value is stored as:
1363
+ *
1364
+ * | Cell | Symbol |
1365
+ * | --- | --- |
1366
+ * | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
1367
+ * | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
1368
+ * | any other finite number | pure double, type 2 |
1369
+ * | a string | pure string, type 4 |
1370
+ * | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
1371
+ * | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
1372
+ *
1373
+ * A number is a pure number, with no text. It used to be written as a dual whose text was
1374
+ * `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
1375
+ * file Qlik wrote with pure numbers came back out of a read and a write with every one of them
1376
+ * turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
1377
+ * A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
1378
+ * UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
1379
+ * an unpaired surrogate has no UTF-8 encoding at all.
601
1380
  *
602
- * Algorithm:
603
- * 1. Initialize a Set for each column to collect unique values
604
- * 2. Single pass through all data rows, adding values to corresponding Sets
605
- * 3. Convert Sets to arrays and create QvdSymbol instances
606
- * 4. Serialize symbols to binary format and update metadata
1381
+ * A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
1382
+ * symbol with the first text in row order, and a plain number with the same number as a dual joins
1383
+ * the dual, in either order - so no text a caller supplied is lost to a plain number that came
1384
+ * first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
607
1385
  *
608
- * This approach provides:
609
- * - 80-90% performance improvement for large datasets (100K+ rows)
610
- * - Better cache locality (process all columns in one data traversal)
611
- * - Lower memory pressure (no intermediate arrays per column)
1386
+ * A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
1387
+ * string read as a number. Its record says what each such cell stands for, so the frame writes back
1388
+ * the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
1389
+ * rows the cells were moved to. A cell the record maps to more than one stored value is refused.
1390
+ *
1391
+ * The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
1392
+ * for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
1393
+ * repository; `benchmarks/` measures what the writer costs now, which is the useful number.
612
1394
  *
613
1395
  * @private
614
1396
  */
615
1397
  _buildSymbolTable() {
616
- this._symbolTable = [];
1398
+ this._symbolCounts = [];
1399
+ this._symbolFacts = [];
617
1400
  this._symbolTableMetadata = [];
618
1401
  this._symbolIndexByValue = [];
619
1402
  if (this._df.columns.length === 0) {
@@ -626,33 +1409,144 @@ var init_QvdFileWriter = __esm({
626
1409
  const data = this._df.data;
627
1410
  const numColumns = columns.length;
628
1411
  const numRows = data.length;
1412
+ validateColumnNames(columns, this._path);
1413
+ const record = normaliseStoredSymbols(this._df.storedSymbols);
629
1414
  this._emitProgress("symbol-table", 0, numColumns);
630
- const indexByValue = columns.map(() => /* @__PURE__ */ new Map());
1415
+ const state = columns.map((name) => {
1416
+ const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
1417
+ const byKey = /* @__PURE__ */ new Map();
1418
+ let byValue = null;
1419
+ let textByNumber = null;
1420
+ if (entry !== null && valuesAreNumbers(entry)) {
1421
+ textByNumber = firstTextByValue(entry);
1422
+ } else if (entry !== null) {
1423
+ byValue = /* @__PURE__ */ new Map();
1424
+ for (let index = 0; index < entry.values.length; index++) {
1425
+ const found = byValue.get(entry.values[index]);
1426
+ if (found === void 0) {
1427
+ byValue.set(entry.values[index], index);
1428
+ } else if (typeof found === "number") {
1429
+ byValue.set(entry.values[index], [found, index]);
1430
+ } else {
1431
+ found.push(index);
1432
+ }
1433
+ }
1434
+ }
1435
+ return {
1436
+ name,
1437
+ keys: [],
1438
+ texts: [],
1439
+ byKey,
1440
+ // A cell is its own key without a record entry, and with one whose values are their numbers, so
1441
+ // one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
1442
+ byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
1443
+ byObject: /* @__PURE__ */ new Map(),
1444
+ entry,
1445
+ textByNumber,
1446
+ byValue,
1447
+ facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
1448
+ };
1449
+ });
1450
+ const byCells = state.map((column) => column.byCell);
1451
+ const byObjects = state.map((column) => column.byObject);
631
1452
  const containsNull = columns.map(() => false);
632
1453
  for (let row = 0; row < numRows; row++) {
633
1454
  const values = data[row];
1455
+ if (values !== null && values !== void 0 && !Array.isArray(values)) {
1456
+ throw new exports.QvdValidationError("Each row must be an array of values", {
1457
+ row,
1458
+ type: typeof values,
1459
+ file: this._path,
1460
+ stage: "buildSymbolTable"
1461
+ });
1462
+ }
1463
+ if (values !== null && values !== void 0 && values.length > numColumns) {
1464
+ throw new exports.QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
1465
+ row,
1466
+ values: values.length,
1467
+ fields: numColumns,
1468
+ file: this._path,
1469
+ stage: "buildSymbolTable"
1470
+ });
1471
+ }
634
1472
  for (let column = 0; column < numColumns; column++) {
635
1473
  const value = values?.[column];
636
1474
  if (value === null || value === void 0) {
637
1475
  containsNull[column] = true;
638
1476
  continue;
639
1477
  }
640
- const map = indexByValue[column];
641
- if (!map.has(value)) {
642
- map.set(value, map.size);
1478
+ if (typeof value === "object") {
1479
+ if (!byObjects[column].has(value)) {
1480
+ slotForObject(state[column], value, row, this._path);
1481
+ }
1482
+ continue;
1483
+ }
1484
+ if (byCells[column].has(value)) {
1485
+ continue;
1486
+ }
1487
+ if (typeof value === "number") {
1488
+ if (numberProblem(value) !== null) {
1489
+ checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
1490
+ }
1491
+ } else if (typeof value === "string") {
1492
+ if (textProblem(value) !== null) {
1493
+ checkText(value, "A string value", {
1494
+ column: columns[column],
1495
+ row,
1496
+ file: this._path,
1497
+ stage: "buildSymbolTable"
1498
+ });
1499
+ }
1500
+ } else {
1501
+ refuseCell(value, columns[column], row, this._path);
643
1502
  }
1503
+ byCells[column].set(value, slotForCell(state[column], value, row, this._path));
644
1504
  }
645
1505
  }
646
1506
  const columnBuffers = [];
647
1507
  let symbolsOffset = 0;
648
1508
  for (let column = 0; column < numColumns; column++) {
649
- const symbols = Array.from(indexByValue[column].keys(), (value) => _QvdFileWriter._convertRawToSymbol(value));
650
- const columnBuffer = Buffer.concat(symbols.map((symbol) => symbol.toByteRepresentation()));
1509
+ const { keys, texts } = state[column];
1510
+ const kinds = new Uint8Array(keys.length);
1511
+ let byteLength = 0;
1512
+ for (let slot = 0; slot < keys.length; slot++) {
1513
+ const key = keys[slot];
1514
+ const text = texts[slot];
1515
+ if (typeof key === "number") {
1516
+ if (numberProblem(key) !== null) {
1517
+ checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
1518
+ }
1519
+ if (text !== null && textProblem(text) !== null) {
1520
+ checkText(text, "The text of a dual value", {
1521
+ column: columns[column],
1522
+ file: this._path,
1523
+ stage: "buildSymbolTable"
1524
+ });
1525
+ }
1526
+ kinds[slot] = kindOf(key, text);
1527
+ byteLength += symbolByteLength(kinds[slot], key, text);
1528
+ } else {
1529
+ kinds[slot] = kindOf(null, key);
1530
+ byteLength += symbolByteLength(kinds[slot], null, key);
1531
+ }
1532
+ }
1533
+ const columnBuffer = Buffer.allocUnsafe(byteLength);
1534
+ let offset = 0;
1535
+ for (let slot = 0; slot < keys.length; slot++) {
1536
+ const key = keys[slot];
1537
+ offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
1538
+ }
1539
+ assert2__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
651
1540
  columnBuffers.push(columnBuffer);
652
- this._symbolTableMetadata?.push([symbolsOffset, columnBuffer.length, containsNull[column]]);
653
- this._symbolTable?.push(symbols);
654
- this._symbolIndexByValue.push(indexByValue[column]);
655
- symbolsOffset += columnBuffer.length;
1541
+ this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
1542
+ this._symbolCounts?.push(keys.length);
1543
+ this._symbolFacts?.push(state[column].facts);
1544
+ this._symbolIndexByValue?.push({
1545
+ byCell: state[column].byCell,
1546
+ byKey: state[column].byKey,
1547
+ byObject: state[column].byObject
1548
+ });
1549
+ symbolsOffset += byteLength;
656
1550
  this._emitProgress("symbol-table", column + 1, numColumns);
657
1551
  }
658
1552
  this._symbolBuffer = Buffer.concat(columnBuffers);
@@ -681,7 +1575,7 @@ var init_QvdFileWriter = __esm({
681
1575
  * @private
682
1576
  */
683
1577
  _buildIndexTable() {
684
- assert2__default.default(this._symbolTable, "The QVD file symbol table has not been built.");
1578
+ assert2__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
685
1579
  assert2__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
686
1580
  assert2__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
687
1581
  this._indexTableMetadata = [];
@@ -695,20 +1589,24 @@ var init_QvdFileWriter = __esm({
695
1589
  let totalBits = 0;
696
1590
  for (let column = 0; column < numColumns; column++) {
697
1591
  const fieldContainsNull = this._symbolTableMetadata[column][2];
698
- const symbolCount = this._symbolTable[column].length;
1592
+ const symbolCount = this._symbolCounts[column];
699
1593
  const nullShift = fieldContainsNull ? 2 : 0;
700
1594
  const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
701
1595
  const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
702
- for (const index of this._symbolIndexByValue[column].values()) {
703
- if (index + nullShift > maxStoredIndex) {
704
- throw new exports.QvdValidationError("The symbol table and the index table are out of sync", {
705
- field: columns[column],
706
- storedIndex: index + nullShift,
707
- maxStoredIndex,
708
- symbolCount,
709
- file: this._path,
710
- stage: "buildIndexTable"
711
- });
1596
+ const lookup = this._symbolIndexByValue[column];
1597
+ const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
1598
+ for (const map of maps) {
1599
+ for (const index of map.values()) {
1600
+ if (index + nullShift > maxStoredIndex) {
1601
+ throw new exports.QvdValidationError("The symbol table and the index table are out of sync", {
1602
+ field: columns[column],
1603
+ storedIndex: index + nullShift,
1604
+ maxStoredIndex,
1605
+ symbolCount,
1606
+ file: this._path,
1607
+ stage: "buildIndexTable"
1608
+ });
1609
+ }
712
1610
  }
713
1611
  }
714
1612
  layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
@@ -719,6 +1617,9 @@ var init_QvdFileWriter = __esm({
719
1617
  this._recordByteSize = recordByteSize;
720
1618
  this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
721
1619
  const progressInterval = Math.max(1, Math.floor(numRows / 100));
1620
+ const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
1621
+ const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
1622
+ const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
722
1623
  for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
723
1624
  const values = data[row];
724
1625
  for (let column = 0; column < numColumns; column++) {
@@ -726,7 +1627,7 @@ var init_QvdFileWriter = __esm({
726
1627
  if (value === null || value === void 0) {
727
1628
  continue;
728
1629
  }
729
- const index = this._symbolIndexByValue[column].get(value);
1630
+ const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
730
1631
  if (index === void 0) {
731
1632
  throw new exports.QvdValidationError("A value is missing from the symbol table", {
732
1633
  field: columns[column],
@@ -743,29 +1644,6 @@ var init_QvdFileWriter = __esm({
743
1644
  }
744
1645
  this._symbolIndexByValue = null;
745
1646
  }
746
- /**
747
- * Converts a raw value/literal to a QVD symbol.
748
- *
749
- * @param {any} raw The raw value/literal to convert.
750
- * @return {QvdSymbol|null} The converted QVD symbol.
751
- */
752
- static _convertRawToSymbol(raw) {
753
- if (raw === null || raw === void 0) {
754
- return null;
755
- }
756
- const INT32_MIN = -2147483648;
757
- const INT32_MAX = 2147483647;
758
- const isInteger = typeof raw === "number" && Number.isInteger(raw);
759
- const isFloat = typeof raw === "number" && !Number.isInteger(raw);
760
- const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
761
- if (isInteger && isWithinInt32Range) {
762
- return exports.QvdSymbol.fromDualIntValue(raw, raw.toString());
763
- } else if (isFloat || isInteger && !isWithinInt32Range) {
764
- return exports.QvdSymbol.fromDualDoubleValue(raw, raw.toString());
765
- } else {
766
- return exports.QvdSymbol.fromStringValue(raw);
767
- }
768
- }
769
1647
  /**
770
1648
  * Persists the data frame to a QVD file.
771
1649
  */
@@ -819,11 +1697,12 @@ function estimateRowMemory(rows, columnCount) {
819
1697
  }
820
1698
  return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
821
1699
  }
822
- function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
1700
+ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
823
1701
  const FULL_PARSE_OVERHEAD = 6;
824
1702
  const MINIMAL_OVERHEAD = 0.01;
825
1703
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
826
- const rowMemory = materialisesRows ? estimateRowMemory(rowsToLoad, columnCount) : BASE_BYTES;
1704
+ const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
1705
+ const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
827
1706
  if (maxRows === null || maxRows >= totalRows) {
828
1707
  return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
829
1708
  }
@@ -833,18 +1712,44 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
833
1712
  const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
834
1713
  return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
835
1714
  }
836
- function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true) {
837
- if (estimateMemoryUsage(symbolTableSize, totalRows, totalRows, columnCount, materialisesRows) <= budget) {
1715
+ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
1716
+ const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
1717
+ if (costOf(totalRows) <= budget) {
838
1718
  return totalRows;
839
1719
  }
840
- if (estimateMemoryUsage(symbolTableSize, 0, totalRows, columnCount, materialisesRows) > budget) {
1720
+ if (costOf(0) > budget) {
841
1721
  return 0;
842
1722
  }
843
1723
  let low = 0;
844
1724
  let high = totalRows;
845
1725
  while (high - low > 1) {
846
1726
  const mid = Math.floor((low + high) / 2);
847
- if (estimateMemoryUsage(symbolTableSize, mid, totalRows, columnCount, materialisesRows) <= budget) {
1727
+ if (costOf(mid) <= budget) {
1728
+ low = mid;
1729
+ } else {
1730
+ high = mid;
1731
+ }
1732
+ }
1733
+ return low;
1734
+ }
1735
+ function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
1736
+ const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
1737
+ const fits = /* @__PURE__ */ __name((chunk) => {
1738
+ const live = Math.min(chunk * liveRowsPerChunk, covered);
1739
+ const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
1740
+ return cost <= budget;
1741
+ }, "fits");
1742
+ if (fits(covered)) {
1743
+ return covered;
1744
+ }
1745
+ if (!fits(1)) {
1746
+ return 0;
1747
+ }
1748
+ let low = 1;
1749
+ let high = covered;
1750
+ while (high - low > 1) {
1751
+ const mid = Math.floor((low + high) / 2);
1752
+ if (fits(mid)) {
848
1753
  low = mid;
849
1754
  } else {
850
1755
  high = mid;
@@ -852,7 +1757,7 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
852
1757
  }
853
1758
  return low;
854
1759
  }
855
- function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true) {
1760
+ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
856
1761
  if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
857
1762
  throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
858
1763
  }
@@ -861,12 +1766,16 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
861
1766
  }
862
1767
  const budget = getMemoryBudget();
863
1768
  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
864
- const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
865
- const externalMemory = estimateExternalMemory(rowsToLoad, columnCount);
1769
+ const rowsLive = live === null ? null : live.rows;
1770
+ const liveRowsPerChunk = live === null ? 1 : live.perChunk;
1771
+ const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
1772
+ const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
1773
+ const externalMemory = estimateExternalMemory(liveRows, columnCount);
866
1774
  const bounded = budget.candidates.map((candidate) => {
867
1775
  const heapOnly = candidate.source === "V8 heap limit";
868
1776
  return {
869
1777
  ...candidate,
1778
+ heapOnly,
870
1779
  needs: heapOnly ? heapMemory : heapMemory + externalMemory,
871
1780
  allowed: candidate.bytes * safetyFactor,
872
1781
  bounds: heapOnly ? "the V8 heap" : "the whole process"
@@ -882,12 +1791,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
882
1791
  const estimatedMemory = binding ? binding.needs : heapMemory;
883
1792
  const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
884
1793
  if (binding) {
1794
+ const includeExternal = !binding.heapOnly;
885
1795
  const recommendedMaxRows = recommendedRowsFor(
886
1796
  maxAllowedMemory,
887
1797
  symbolTableSize,
888
1798
  totalRows,
889
1799
  columnCount,
890
- materialisesRows
1800
+ materialisesRows,
1801
+ includeExternal
891
1802
  );
892
1803
  const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
893
1804
  const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
@@ -899,15 +1810,27 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
899
1810
  const limitingScope = binding.bounds;
900
1811
  const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
901
1812
  const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
902
- const nothingFits = recommendedMaxRows === 0;
903
1813
  const containerBound = binding.source === "container memory limit";
1814
+ const chunked = rowsLive !== null;
1815
+ const recommendedChunk = chunked ? recommendedChunkFor(
1816
+ maxAllowedMemory,
1817
+ symbolTableSize,
1818
+ maxRows,
1819
+ totalRows,
1820
+ columnCount,
1821
+ liveRowsPerChunk,
1822
+ includeExternal
1823
+ ) : 0;
1824
+ const knob = chunked ? "chunkSize" : "limit";
1825
+ const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
1826
+ const nothingFits = recommendedValue === 0;
904
1827
  let advice;
905
1828
  if (nothingFits) {
906
- advice = `No row count fits this budget - the symbol table alone exceeds it, so maxRows cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
1829
+ advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
907
1830
  } else if (containerBound) {
908
- advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`;
1831
+ advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
909
1832
  } else {
910
- advice = `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
1833
+ advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
911
1834
  }
912
1835
  throw new exports.QvdValidationError(
913
1836
  `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
@@ -927,32 +1850,55 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
927
1850
  columnCount,
928
1851
  totalRows,
929
1852
  maxRows,
930
- recommendedMaxRows
1853
+ recommendedMaxRows,
1854
+ // Only present when a chunk size is what overflowed, so a caller cannot mistake one
1855
+ // recommendation for the other.
1856
+ ...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
931
1857
  }
932
1858
  );
933
1859
  }
934
1860
  }
1861
+ function formatCount(value) {
1862
+ return value.toLocaleString("en-US");
1863
+ }
935
1864
  function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
936
1865
  const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
937
- if (symbolTableSize > LARGE_SYMBOL_TABLE_WARNING && (maxRows === null || maxRows >= totalRows)) {
938
- const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
939
- const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
940
- const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
941
- const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
942
- console.warn(
943
- `\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
944
- );
1866
+ if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
1867
+ return;
945
1868
  }
1869
+ const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
1870
+ const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
1871
+ if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
1872
+ return;
1873
+ }
1874
+ const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
1875
+ const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
1876
+ const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
1877
+ console.warn(
1878
+ `\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
1879
+ );
946
1880
  }
947
1881
  var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
948
1882
  var init_memoryUtils = __esm({
949
1883
  "src/util/memoryUtils.js"() {
950
1884
  init_QvdErrors();
1885
+ __name(getHeapLimit, "getHeapLimit");
1886
+ __name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
1887
+ __name(getMemoryBudget, "getMemoryBudget");
951
1888
  HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
952
1889
  MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
1890
+ __name(usableOldSpaceLimit, "usableOldSpaceLimit");
953
1891
  BASE_BYTES = 16 * 1024 * 1024;
954
1892
  ROW_BASE_BYTES = 72;
955
1893
  PER_CELL_BYTES = 8;
1894
+ __name(estimateExternalMemory, "estimateExternalMemory");
1895
+ __name(estimateRowMemory, "estimateRowMemory");
1896
+ __name(estimateMemoryUsage, "estimateMemoryUsage");
1897
+ __name(recommendedRowsFor, "recommendedRowsFor");
1898
+ __name(recommendedChunkFor, "recommendedChunkFor");
1899
+ __name(validateMemoryAvailability, "validateMemoryAvailability");
1900
+ __name(formatCount, "formatCount");
1901
+ __name(warnLargeSymbolTable, "warnLargeSymbolTable");
956
1902
  }
957
1903
  });
958
1904
 
@@ -983,7 +1929,7 @@ function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
983
1929
  const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
984
1930
  const heapMB = Math.round(heapLimit / 1024 / 1024);
985
1931
  throw new exports.QvdValidationError(
986
- `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
1932
+ `Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
987
1933
  {
988
1934
  file: filePath,
989
1935
  symbolTableSize: symbolTableLength,
@@ -1057,7 +2003,7 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
1057
2003
  });
1058
2004
  }
1059
2005
  }
1060
- function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
2006
+ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
1061
2007
  if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
1062
2008
  throw new exports.QvdCorruptedError("Invalid record byte size", {
1063
2009
  recordSize,
@@ -1119,23 +2065,28 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
1119
2065
  }
1120
2066
  }
1121
2067
  const requiredIndexBytes = rowsToLoad * recordSize;
1122
- if (indexTableOffset + requiredIndexBytes > bufferLength) {
2068
+ const bufferRecordStart = (windowFirstRow - bufferFirstRow) * recordSize;
2069
+ if (indexTableOffset + bufferRecordStart + requiredIndexBytes > bufferLength) {
1123
2070
  throw new exports.QvdCorruptedError("Index table truncated", {
1124
2071
  indexTableOffset,
1125
2072
  requiredBytes: requiredIndexBytes,
1126
- availableBytes: Math.max(0, bufferLength - indexTableOffset),
2073
+ availableBytes: Math.max(0, bufferLength - indexTableOffset - bufferRecordStart),
1127
2074
  rowsToLoad,
2075
+ windowFirstRow,
2076
+ bufferFirstRow,
1128
2077
  recordSize,
1129
2078
  bufferSize: bufferLength,
1130
2079
  file: filePath,
1131
2080
  stage: "parseIndexTable"
1132
2081
  });
1133
2082
  }
1134
- if (indexTableLength < requiredIndexBytes) {
2083
+ const requiredTableBytes = (windowFirstRow + rowsToLoad) * recordSize;
2084
+ if (indexTableLength < requiredTableBytes) {
1135
2085
  throw new exports.QvdCorruptedError("Index table length smaller than required", {
1136
2086
  indexTableLength,
1137
- requiredBytes: requiredIndexBytes,
2087
+ requiredBytes: requiredTableBytes,
1138
2088
  rowsToLoad,
2089
+ windowFirstRow,
1139
2090
  recordSize,
1140
2091
  file: filePath,
1141
2092
  stage: "parseIndexTable"
@@ -1196,262 +2147,235 @@ var init_validationUtils = __esm({
1196
2147
  init_QvdErrors();
1197
2148
  init_memoryUtils();
1198
2149
  init_bitUtils();
2150
+ __name(validateHeaderStructure, "validateHeaderStructure");
2151
+ __name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
2152
+ __name(validateSymbolTableSize, "validateSymbolTableSize");
2153
+ __name(validateFieldMetadata, "validateFieldMetadata");
2154
+ __name(validateRecordCount, "validateRecordCount");
2155
+ __name(validateIndexTableMetadata, "validateIndexTableMetadata");
2156
+ __name(validateFieldBitMetadata, "validateFieldBitMetadata");
1199
2157
  }
1200
2158
  });
1201
2159
 
1202
2160
  // src/util/symbolParser.js
1203
- function parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1204
- if (pointer + 4 > bufferLength) {
1205
- throw new exports.QvdCorruptedError("Buffer overflow reading integer symbol", {
1206
- field: fieldName,
1207
- pointer,
1208
- bufferSize: bufferLength,
1209
- file: filePath,
1210
- stage: "parseSymbolTable"
1211
- });
1212
- }
1213
- const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
1214
- const value = Buffer.from(byteData).readIntLE(0, byteData.length);
1215
- return { symbol: exports.QvdSymbol.fromIntValue(value), bytesRead: 4 };
1216
- }
1217
- function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1218
- if (pointer + 8 > bufferLength) {
1219
- throw new exports.QvdCorruptedError("Buffer overflow reading double symbol", {
2161
+ function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
2162
+ const bufferLength = symbolBuffer.length;
2163
+ const found = symbolBuffer.indexOf(0, from);
2164
+ if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
2165
+ throw new exports.QvdCorruptedError(`${kind} exceeds maximum length`, {
1220
2166
  field: fieldName,
1221
- pointer,
1222
- bufferSize: bufferLength,
1223
- file: filePath,
1224
- stage: "parseSymbolTable"
1225
- });
1226
- }
1227
- const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
1228
- const value = Buffer.from(byteData).readDoubleLE(0);
1229
- return { symbol: exports.QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
1230
- }
1231
- function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1232
- const startPointer = pointer;
1233
- const maxStringLength = 1048576;
1234
- let stringLength = 0;
1235
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1236
- if (stringLength >= maxStringLength) {
1237
- throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
1238
- field: fieldName,
1239
- maxLength: maxStringLength,
1240
- file: filePath,
1241
- stage: "parseSymbolTable"
1242
- });
1243
- }
1244
- pointer++;
1245
- stringLength++;
1246
- }
1247
- if (pointer >= bufferLength) {
1248
- throw new exports.QvdCorruptedError("String symbol not null-terminated", {
1249
- field: fieldName,
1250
- pointer,
1251
- bufferSize: bufferLength,
2167
+ maxLength: MAX_TEXT_BYTES,
1252
2168
  file: filePath,
1253
2169
  stage: "parseSymbolTable"
1254
2170
  });
1255
2171
  }
1256
- const value = symbolBuffer.subarray(startPointer, pointer).toString("utf-8");
1257
- return { symbol: exports.QvdSymbol.fromStringValue(value), bytesRead: pointer - startPointer + 1 };
1258
- }
1259
- function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1260
- const startPointer = pointer;
1261
- const maxStringLength = 1048576;
1262
- let stringLength = 0;
1263
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1264
- if (stringLength >= maxStringLength) {
1265
- throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
1266
- field: fieldName,
1267
- maxLength: maxStringLength,
1268
- file: filePath,
1269
- stage: "parseSymbolTable"
1270
- });
1271
- }
1272
- pointer++;
1273
- stringLength++;
1274
- }
1275
- if (pointer >= bufferLength) {
1276
- throw new exports.QvdCorruptedError("String symbol not null-terminated", {
2172
+ if (found === -1) {
2173
+ throw new exports.QvdCorruptedError(`${kind} not null-terminated`, {
1277
2174
  field: fieldName,
1278
- pointer,
2175
+ pointer: bufferLength,
1279
2176
  bufferSize: bufferLength,
1280
2177
  file: filePath,
1281
2178
  stage: "parseSymbolTable"
1282
2179
  });
1283
2180
  }
1284
- return pointer - startPointer + 1;
2181
+ return found;
1285
2182
  }
1286
- function parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1287
- if (pointer + 4 > bufferLength) {
1288
- throw new exports.QvdCorruptedError("Buffer overflow reading dual integer symbol", {
1289
- field: fieldName,
1290
- pointer,
1291
- bufferSize: bufferLength,
1292
- file: filePath,
1293
- stage: "parseSymbolTable"
1294
- });
1295
- }
1296
- const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
1297
- const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
1298
- pointer += 4;
1299
- const stringStart = pointer;
1300
- const maxStringLength = 1048576;
1301
- let stringLength = 0;
1302
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1303
- if (stringLength >= maxStringLength) {
1304
- throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
1305
- field: fieldName,
1306
- maxLength: maxStringLength,
1307
- file: filePath,
1308
- stage: "parseSymbolTable"
1309
- });
1310
- }
1311
- pointer++;
1312
- stringLength++;
1313
- }
1314
- if (pointer >= bufferLength) {
1315
- throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
1316
- field: fieldName,
1317
- pointer,
1318
- bufferSize: bufferLength,
1319
- file: filePath,
1320
- stage: "parseSymbolTable"
1321
- });
1322
- }
1323
- const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
1324
- return { symbol: exports.QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
2183
+ function overflow(message, pointer, bufferLength, fieldName, filePath) {
2184
+ throw new exports.QvdCorruptedError(message, {
2185
+ field: fieldName,
2186
+ pointer,
2187
+ bufferSize: bufferLength,
2188
+ file: filePath,
2189
+ stage: "parseSymbolTable"
2190
+ });
1325
2191
  }
1326
- function parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
1327
- if (pointer + 8 > bufferLength) {
1328
- throw new exports.QvdCorruptedError("Buffer overflow reading dual double symbol", {
1329
- field: fieldName,
1330
- pointer,
1331
- bufferSize: bufferLength,
1332
- file: filePath,
1333
- stage: "parseSymbolTable"
1334
- });
1335
- }
1336
- const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
1337
- const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
1338
- pointer += 8;
1339
- const stringStart = pointer;
1340
- const maxStringLength = 1048576;
1341
- let stringLength = 0;
1342
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1343
- if (stringLength >= maxStringLength) {
1344
- throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
1345
- field: fieldName,
1346
- maxLength: maxStringLength,
1347
- file: filePath,
1348
- stage: "parseSymbolTable"
1349
- });
2192
+ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
2193
+ const bufferLength = symbolBuffer.length;
2194
+ const numbers = [];
2195
+ const texts = [];
2196
+ let pointer = start;
2197
+ while (pointer < end) {
2198
+ const typeByte = symbolBuffer[pointer++];
2199
+ const decode = keep === null || keep.has(numbers.length);
2200
+ let number = null;
2201
+ let text = null;
2202
+ switch (typeByte) {
2203
+ case 1: {
2204
+ if (decode) {
2205
+ if (pointer + 4 > bufferLength) {
2206
+ overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
2207
+ }
2208
+ number = symbolBuffer.readInt32LE(pointer);
2209
+ }
2210
+ pointer += 4;
2211
+ break;
2212
+ }
2213
+ case 2: {
2214
+ if (decode) {
2215
+ if (pointer + 8 > bufferLength) {
2216
+ overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
2217
+ }
2218
+ number = symbolBuffer.readDoubleLE(pointer);
2219
+ }
2220
+ pointer += 8;
2221
+ break;
2222
+ }
2223
+ case 4: {
2224
+ const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
2225
+ if (decode) {
2226
+ text = symbolBuffer.toString("utf8", pointer, terminator);
2227
+ }
2228
+ pointer = terminator + 1;
2229
+ break;
2230
+ }
2231
+ case 5:
2232
+ case 6: {
2233
+ const numberBytes = typeByte === 5 ? 4 : 8;
2234
+ if (pointer + numberBytes > bufferLength) {
2235
+ const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
2236
+ overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
2237
+ }
2238
+ const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
2239
+ if (decode) {
2240
+ number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
2241
+ text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
2242
+ }
2243
+ pointer = terminator + 1;
2244
+ break;
2245
+ }
2246
+ default: {
2247
+ throw new exports.QvdParseError("Unknown symbol type byte", {
2248
+ typeByte: typeByte.toString(16),
2249
+ offset: pointer - 1,
2250
+ file: filePath,
2251
+ stage: "parseSymbolTable"
2252
+ });
2253
+ }
1350
2254
  }
1351
- pointer++;
1352
- stringLength++;
1353
- }
1354
- if (pointer >= bufferLength) {
1355
- throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
1356
- field: fieldName,
1357
- pointer,
1358
- bufferSize: bufferLength,
1359
- file: filePath,
1360
- stage: "parseSymbolTable"
1361
- });
2255
+ numbers.push(number);
2256
+ texts.push(text);
1362
2257
  }
1363
- const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
1364
- return {
1365
- symbol: exports.QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
1366
- bytesRead: pointer - (stringStart - 8) + 1
1367
- };
2258
+ return { numbers, texts };
1368
2259
  }
1369
- function skipDualSymbol(symbolBuffer, pointer, bufferLength, numericBytes, fieldName, filePath) {
1370
- if (pointer + numericBytes > bufferLength) {
1371
- throw new exports.QvdCorruptedError(`Buffer overflow reading dual symbol`, {
1372
- field: fieldName,
1373
- pointer,
1374
- bufferSize: bufferLength,
1375
- file: filePath,
1376
- stage: "parseSymbolTable"
1377
- });
2260
+ var MAX_TEXT_BYTES;
2261
+ var init_symbolParser = __esm({
2262
+ "src/util/symbolParser.js"() {
2263
+ init_QvdErrors();
2264
+ MAX_TEXT_BYTES = 1048576;
2265
+ __name(textEnd, "textEnd");
2266
+ __name(overflow, "overflow");
2267
+ __name(parseFieldSymbols, "parseFieldSymbols");
1378
2268
  }
1379
- const startPointer = pointer;
1380
- pointer += numericBytes;
1381
- const maxStringLength = 1048576;
1382
- let stringLength = 0;
1383
- while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
1384
- if (stringLength >= maxStringLength) {
1385
- throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
1386
- field: fieldName,
1387
- maxLength: maxStringLength,
1388
- file: filePath,
1389
- stage: "parseSymbolTable"
1390
- });
2269
+ });
2270
+
2271
+ // src/util/resolveSymbols.js
2272
+ function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
2273
+ const length = symbols.numbers.length;
2274
+ const values = new Array(length);
2275
+ const entryValues = [];
2276
+ const numbers = [];
2277
+ const texts = [];
2278
+ let pure = 0;
2279
+ let partial = false;
2280
+ for (let index = 0; index < length; index++) {
2281
+ const text = symbols.texts[index];
2282
+ const number = symbols.numbers[index];
2283
+ if (text === null) {
2284
+ values[index] = number === null ? void 0 : number;
2285
+ if (number !== null) pure++;
2286
+ continue;
2287
+ }
2288
+ if (number === null) {
2289
+ if (coerce && isNumericText(text)) {
2290
+ values[index] = Number(text);
2291
+ entryValues.push(values[index]);
2292
+ numbers.push(null);
2293
+ texts.push(text);
2294
+ partial = true;
2295
+ } else {
2296
+ values[index] = text;
2297
+ pure++;
2298
+ }
2299
+ continue;
2300
+ }
2301
+ partial = true;
2302
+ if (mode === "both") {
2303
+ values[index] = dualFromSymbol(number, text);
2304
+ continue;
1391
2305
  }
1392
- pointer++;
1393
- stringLength++;
2306
+ values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
2307
+ entryValues.push(values[index]);
2308
+ numbers.push(number);
2309
+ texts.push(text);
1394
2310
  }
1395
- if (pointer >= bufferLength) {
1396
- throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
1397
- field: fieldName,
1398
- pointer,
1399
- bufferSize: bufferLength,
1400
- file: filePath,
1401
- stage: "parseSymbolTable"
2311
+ let entry = null;
2312
+ if (entryValues.length > 0) {
2313
+ entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
2314
+ field,
2315
+ values: Object.freeze(entryValues),
2316
+ numbers: Object.freeze(numbers),
2317
+ texts: Object.freeze(texts)
1402
2318
  });
1403
2319
  }
1404
- return pointer - startPointer + 1;
2320
+ return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
1405
2321
  }
1406
- function parseSymbol(typeByte, symbolBuffer, pointer, bufferLength, fieldName, filePath, shouldParse) {
1407
- switch (typeByte) {
1408
- case 1: {
1409
- if (shouldParse) {
1410
- return parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1411
- }
1412
- return { symbol: null, bytesRead: 4 };
1413
- }
1414
- case 2: {
1415
- if (shouldParse) {
1416
- return parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1417
- }
1418
- return { symbol: null, bytesRead: 8 };
2322
+ function collides(symbols, values, recorded) {
2323
+ for (let index = 0; index < values.length; index++) {
2324
+ if (!isPure(symbols, index, values[index])) {
2325
+ continue;
1419
2326
  }
1420
- case 4: {
1421
- if (shouldParse) {
1422
- return parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1423
- }
1424
- const bytesRead = skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1425
- return { symbol: null, bytesRead };
2327
+ if (recorded.has(values[index])) {
2328
+ return true;
1426
2329
  }
1427
- case 5: {
1428
- if (shouldParse) {
1429
- return parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1430
- }
1431
- const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 4, fieldName, filePath);
1432
- return { symbol: null, bytesRead };
2330
+ }
2331
+ return false;
2332
+ }
2333
+ function isPure(symbols, index, value) {
2334
+ const text = symbols.texts[index];
2335
+ const number = symbols.numbers[index];
2336
+ return text === null ? number !== null : number === null && value === text;
2337
+ }
2338
+ function collisionEntry(symbols, field, values, mode) {
2339
+ const recorded = /* @__PURE__ */ new Set();
2340
+ const inPassA = values.map((value, index) => {
2341
+ if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
2342
+ return false;
1433
2343
  }
1434
- case 6: {
1435
- if (shouldParse) {
1436
- return parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
1437
- }
1438
- const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
1439
- return { symbol: null, bytesRead };
2344
+ const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
2345
+ if (recordedHere) {
2346
+ recorded.add(values[index]);
1440
2347
  }
1441
- default: {
1442
- throw new exports.QvdParseError("Unknown symbol type byte", {
1443
- typeByte: typeByte.toString(16),
1444
- offset: pointer - 1,
1445
- file: filePath,
1446
- stage: "parseSymbolTable"
1447
- });
2348
+ return recordedHere;
2349
+ });
2350
+ const entryValues = [];
2351
+ const numbers = [];
2352
+ const texts = [];
2353
+ for (let index = 0; index < values.length; index++) {
2354
+ if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
2355
+ entryValues.push(values[index]);
2356
+ numbers.push(symbols.numbers[index]);
2357
+ texts.push(symbols.texts[index]);
1448
2358
  }
1449
2359
  }
2360
+ return Object.freeze({
2361
+ field,
2362
+ values: Object.freeze(entryValues),
2363
+ numbers: Object.freeze(numbers),
2364
+ texts: Object.freeze(texts)
2365
+ });
1450
2366
  }
1451
- var init_symbolParser = __esm({
1452
- "src/util/symbolParser.js"() {
1453
- init_QvdSymbol();
1454
- init_QvdErrors();
2367
+ function symbolHalves(symbols) {
2368
+ return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
2369
+ }
2370
+ var init_resolveSymbols = __esm({
2371
+ "src/util/resolveSymbols.js"() {
2372
+ init_QvdDual();
2373
+ init_cellRules();
2374
+ __name(resolveFieldSymbols, "resolveFieldSymbols");
2375
+ __name(collides, "collides");
2376
+ __name(isPure, "isPure");
2377
+ __name(collisionEntry, "collisionEntry");
2378
+ __name(symbolHalves, "symbolHalves");
1455
2379
  }
1456
2380
  });
1457
2381
 
@@ -1461,20 +2385,49 @@ __export(QvdColumnTable_exports, {
1461
2385
  QvdColumn: () => exports.QvdColumn,
1462
2386
  QvdColumnTable: () => exports.QvdColumnTable
1463
2387
  });
2388
+ function numberOf(value, number) {
2389
+ if (typeof value === "number") {
2390
+ return value;
2391
+ }
2392
+ if (typeof value === "string") {
2393
+ return number ?? NaN;
2394
+ }
2395
+ const dual = asDual(value);
2396
+ return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
2397
+ }
2398
+ function textOf(value) {
2399
+ if (typeof value === "string") {
2400
+ return value;
2401
+ }
2402
+ const dual = asDual(value);
2403
+ return dual !== null && typeof dual.text === "string" ? dual.text : null;
2404
+ }
1464
2405
  exports.QvdColumn = void 0; exports.QvdColumnTable = void 0;
1465
2406
  var init_QvdColumnTable = __esm({
1466
2407
  "src/QvdColumnTable.js"() {
1467
2408
  init_QvdErrors();
2409
+ init_cellRules();
2410
+ init_readOptions();
2411
+ __name(numberOf, "numberOf");
2412
+ __name(textOf, "textOf");
1468
2413
  exports.QvdColumn = class {
2414
+ static {
2415
+ __name(this, "QvdColumn");
2416
+ }
1469
2417
  /**
1470
2418
  * @param {string} name The field name.
1471
2419
  * @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
1472
2420
  * @param {Array<any>} symbols The field's distinct values, indexed by code.
2421
+ * @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
2422
+ * field whose values do not show them all - a dual read as one half, a string read as a number.
2423
+ * Without them, the halves are derived from the values: a string is its own text, a dual has its
2424
+ * own, and a number has none.
1473
2425
  */
1474
- constructor(name, codes, symbols) {
2426
+ constructor(name, codes, symbols, halves = null) {
1475
2427
  this._name = name;
1476
2428
  this._codes = codes;
1477
2429
  this._symbols = symbols;
2430
+ this._halves = halves;
1478
2431
  Object.freeze(this);
1479
2432
  }
1480
2433
  /** @return {string} The field name. */
@@ -1501,6 +2454,9 @@ var init_QvdColumnTable = __esm({
1501
2454
  *
1502
2455
  * One entry per distinct value, not per row: a few thousand entries for a column of millions.
1503
2456
  *
2457
+ * A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
2458
+ * entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
2459
+ *
1504
2460
  * @return {ReadonlyArray<any>} The dictionary.
1505
2461
  */
1506
2462
  get symbols() {
@@ -1524,6 +2480,42 @@ var init_QvdColumnTable = __esm({
1524
2480
  const code = this._codes[row];
1525
2481
  return code < 0 ? null : this._symbols[code];
1526
2482
  }
2483
+ /**
2484
+ * The text of one row: the text Qlik displays for its value.
2485
+ *
2486
+ * A string is its own text and a dual has its own. A value read as one half of a symbol - a date
2487
+ * read as its serial, a string read as a number - has the text the file stores for it, and a pure
2488
+ * number has none.
2489
+ *
2490
+ * @param {number} row The row index.
2491
+ * @return {string|null} The text, or null for NULL and for a number with no text.
2492
+ * @throws {QvdValidationError} If the row is not an integer within the column.
2493
+ */
2494
+ textAt(row) {
2495
+ if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
2496
+ throw new exports.QvdValidationError("Row index out of bounds", {
2497
+ column: this._name,
2498
+ row,
2499
+ length: this._codes.length
2500
+ });
2501
+ }
2502
+ const code = this._codes[row];
2503
+ if (code < 0) {
2504
+ return null;
2505
+ }
2506
+ return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
2507
+ }
2508
+ /**
2509
+ * The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
2510
+ *
2511
+ * @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
2512
+ */
2513
+ symbolTexts() {
2514
+ if (this._halves !== null) {
2515
+ return this._halves.texts;
2516
+ }
2517
+ return Object.freeze(this._symbols.map(textOf));
2518
+ }
1527
2519
  /**
1528
2520
  * Iterates the column's values without materialising it.
1529
2521
  *
@@ -1567,6 +2559,9 @@ var init_QvdColumnTable = __esm({
1567
2559
  * one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
1568
2560
  * codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
1569
2561
  *
2562
+ * A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
2563
+ * `{duals: 'text'}` gives the serial the file stores for it, not NaN.
2564
+ *
1570
2565
  * Scanning `codes` against this is the fastest way to read a column, because both sides are
1571
2566
  * contiguous typed arrays and the dictionary fits in cache:
1572
2567
  *
@@ -1588,8 +2583,7 @@ var init_QvdColumnTable = __esm({
1588
2583
  numericSymbols() {
1589
2584
  const out = new Float64Array(this._symbols.length);
1590
2585
  for (let index = 0; index < this._symbols.length; index++) {
1591
- const value = this._symbols[index];
1592
- out[index] = typeof value === "number" ? value : NaN;
2586
+ out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
1593
2587
  }
1594
2588
  return out;
1595
2589
  }
@@ -1604,7 +2598,8 @@ var init_QvdColumnTable = __esm({
1604
2598
  * @param {Object} [options] Conversion options.
1605
2599
  * @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
1606
2600
  * number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
1607
- * NaN, which is the right choice only when the caller knows the column is numeric.
2601
+ * NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
2602
+ * number here, as it is to `numericSymbols`.
1608
2603
  * @return {Float64Array} One number per row.
1609
2604
  * @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
1610
2605
  */
@@ -1624,6 +2619,11 @@ var init_QvdColumnTable = __esm({
1624
2619
  out[row] = value;
1625
2620
  continue;
1626
2621
  }
2622
+ const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
2623
+ if (!Number.isNaN(number)) {
2624
+ out[row] = number;
2625
+ continue;
2626
+ }
1627
2627
  if (onNonNumeric === "throw") {
1628
2628
  throw new exports.QvdValidationError("Column holds a value that is not a number", {
1629
2629
  column: this._name,
@@ -1639,29 +2639,57 @@ var init_QvdColumnTable = __esm({
1639
2639
  }
1640
2640
  };
1641
2641
  exports.QvdColumnTable = class {
2642
+ static {
2643
+ __name(this, "QvdColumnTable");
2644
+ }
1642
2645
  /**
1643
2646
  * @param {Object} decoded What the reader decoded.
1644
2647
  * @param {Array<string>} decoded.columns Field names, in file order.
1645
2648
  * @param {Array<Int32Array>} decoded.codesByField One code array per field.
1646
2649
  * @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
2650
+ * @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
2651
+ * or null for a field whose values show them.
1647
2652
  * @param {number} decoded.rowCount Rows decoded.
1648
2653
  * @param {any} decoded.metadata The raw QvdTableHeader.
2654
+ * @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
2655
+ * stored-symbol record, as a data frame of the same read carries it.
1649
2656
  * @param {any} decoded.loadStats Statistics about the read.
1650
2657
  */
1651
- constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
2658
+ constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
1652
2659
  this._columns = columns;
1653
2660
  this._codesByField = codesByField;
1654
2661
  this._symbolsByField = symbolsByField;
2662
+ this._halvesByField = halvesByField ?? null;
1655
2663
  this._rowCount = rowCount;
1656
2664
  this._metadata = metadata;
2665
+ this._storedSymbols = storedSymbols ?? null;
1657
2666
  this._loadStats = loadStats;
1658
2667
  }
1659
2668
  /**
1660
2669
  * Reads a QVD file as columns.
1661
2670
  *
2671
+ * Takes the same options as `QvdDataFrame.fromQvd`, with the same meanings - one option
2672
+ * vocabulary for both read paths, because they are two answers about the same file rather than
2673
+ * two features. `{offset, limit}` is how a caller pages through a file columnwise; there is no
2674
+ * columnar `iterate()` because there is nothing for it to bound - a columnar read materialises
2675
+ * no rows, which is the memory chunking exists to cap.
2676
+ *
1662
2677
  * @param {string} path The path to the QVD file.
1663
2678
  * @param {Object} [options] Loading options, with the same meanings they have on `fromQvd`.
1664
- * @param {number|null} [options.maxRows] Maximum rows to decode.
2679
+ * @param {number|null} [options.maxRows] Rows to decode. The older name for `limit`.
2680
+ * @param {number|null} [options.limit] Rows to decode, counting from `offset`.
2681
+ * @param {number} [options.offset] File row to start at.
2682
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
2683
+ * appear. Unselected fields have their symbols skipped entirely.
2684
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
2685
+ * number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
2686
+ * text and `numericSymbols` the number. Anything else throws.
2687
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
2688
+ * number when its text is not blank and `Number(text)` is finite - a string symbol as
2689
+ * `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
2690
+ * Anything but a boolean throws.
2691
+ * @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
2692
+ * @param {AbortSignal} [options.signal] Cancels the read.
1665
2693
  * @param {string} [options.allowedDir] Directory the path must resolve inside.
1666
2694
  * @param {number} [options.memorySafetyFactor] Fraction of the memory budget a load may use.
1667
2695
  * @param {number} [options.symbolFilteringThreshold] Symbol table size above which a limited
@@ -1671,15 +2699,13 @@ var init_QvdColumnTable = __esm({
1671
2699
  static async fromQvd(path3, options = {}) {
1672
2700
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
1673
2701
  const reader = new QvdFileReader2(path3, {
1674
- allowedDir: options.allowedDir,
1675
- memorySafetyFactor: options.memorySafetyFactor,
1676
- symbolFilteringThreshold: options.symbolFilteringThreshold,
2702
+ ...readerOptionsFrom(options),
1677
2703
  // This read builds no rows, so the memory guard must not charge it for them. A columnar
1678
2704
  // read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
1679
2705
  // refused below a 2GB one.
1680
2706
  materialisesRows: false
1681
2707
  });
1682
- return await reader.loadColumnar(options.maxRows !== void 0 ? options.maxRows : null);
2708
+ return await reader.loadColumnar(windowFrom(options));
1683
2709
  }
1684
2710
  /** @return {Array<string>} Field names, in file order. */
1685
2711
  get columns() {
@@ -1701,6 +2727,14 @@ var init_QvdColumnTable = __esm({
1701
2727
  get loadStats() {
1702
2728
  return this._loadStats;
1703
2729
  }
2730
+ /**
2731
+ * The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
2732
+ *
2733
+ * @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
2734
+ */
2735
+ get storedSymbols() {
2736
+ return this._storedSymbols;
2737
+ }
1704
2738
  /**
1705
2739
  * One column.
1706
2740
  *
@@ -1716,7 +2750,12 @@ var init_QvdColumnTable = __esm({
1716
2750
  availableColumns: this._columns
1717
2751
  });
1718
2752
  }
1719
- return new exports.QvdColumn(name, this._codesByField[index], this._symbolsByField[index]);
2753
+ return new exports.QvdColumn(
2754
+ name,
2755
+ this._codesByField[index],
2756
+ this._symbolsByField[index],
2757
+ this._halvesByField?.[index] ?? null
2758
+ );
1720
2759
  }
1721
2760
  };
1722
2761
  }
@@ -1727,7 +2766,17 @@ var QvdFileReader_exports = {};
1727
2766
  __export(QvdFileReader_exports, {
1728
2767
  QvdFileReader: () => exports.QvdFileReader
1729
2768
  });
1730
- var MAX_HEADER_SIZE, READ_CHUNK_SIZE; exports.QvdFileReader = void 0;
2769
+ function closeReadStream(stream) {
2770
+ if (stream.closed) {
2771
+ return Promise.resolve();
2772
+ }
2773
+ return new Promise((resolve) => {
2774
+ stream.once("close", () => resolve());
2775
+ stream.once("error", () => resolve());
2776
+ stream.destroy();
2777
+ });
2778
+ }
2779
+ var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS; exports.QvdFileReader = void 0;
1731
2780
  var init_QvdFileReader = __esm({
1732
2781
  "src/QvdFileReader.js"() {
1733
2782
  init_QvdDataFrame();
@@ -1737,9 +2786,17 @@ var init_QvdFileReader = __esm({
1737
2786
  init_memoryUtils();
1738
2787
  init_validationUtils();
1739
2788
  init_symbolParser();
2789
+ init_readOptions();
2790
+ init_resolveSymbols();
2791
+ init_storedSymbols();
1740
2792
  MAX_HEADER_SIZE = 16 * 1024 * 1024;
1741
2793
  READ_CHUNK_SIZE = 512 * 1024 * 1024;
2794
+ ANALYSIS_SLICE_ROWS = 65536;
2795
+ __name(closeReadStream, "closeReadStream");
1742
2796
  exports.QvdFileReader = class {
2797
+ static {
2798
+ __name(this, "QvdFileReader");
2799
+ }
1743
2800
  /**
1744
2801
  * Constructs a new QVD file parser.
1745
2802
  *
@@ -1750,9 +2807,10 @@ var init_QvdFileReader = __esm({
1750
2807
  * points outside it is rejected. Defaults to the current working directory. To permit
1751
2808
  * an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
1752
2809
  * empty value falls back to the working directory rather than removing the restriction.
1753
- * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a
1754
- * load may use. The budget is the smallest of the V8 heap limit, any container memory limit,
1755
- * and the memory the OS reports as available. Default is 0.3. **Zero disables the memory
2810
+ * @param {number} [options.memorySafetyFactor=0.8] Fraction (0.0-1.0) of the memory budget a
2811
+ * load may use. The budget is the smaller of the V8 heap limit and any container memory limit;
2812
+ * what the OS reports as available is recorded for diagnostics and deliberately not allowed to
2813
+ * bind - see `getMemoryBudget`. Default is 0.8. **Zero disables the memory
1756
2814
  * check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
1757
2815
  * Bun reports its current heap as its heap limit - and for callers who would rather manage
1758
2816
  * memory themselves than trust the estimate.
@@ -1763,41 +2821,123 @@ var init_QvdFileReader = __esm({
1763
2821
  * above which a lazy load switches to the two-pass filtering path. The default of 50MB is
1764
2822
  * the point where the extra analysis pass pays for itself; lower it to use filtering on
1765
2823
  * smaller files, raise it to keep the simpler single-pass read for longer.
2824
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
2825
+ * appear. Null reads every field, in file order. An unknown or repeated name is refused.
2826
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
2827
+ * text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
2828
+ * value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
2829
+ * `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
2830
+ * `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
2831
+ * the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
2832
+ * throws a `QvdValidationError`.
2833
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
2834
+ * reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
2835
+ * every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
2836
+ * dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
2837
+ * dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
2838
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
2839
+ * read proceeds - the same shape `QvdFileWriter` emits.
2840
+ * @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
2841
+ * `signal.reason`, exactly as `signal.throwIfAborted()` does.
1766
2842
  */
1767
2843
  constructor(filePath, options = {}) {
1768
2844
  const {
1769
2845
  allowedDir,
1770
2846
  memorySafetyFactor = 0.8,
1771
2847
  symbolFilteringThreshold = 50 * 1024 * 1024,
1772
- materialisesRows = true
2848
+ materialisesRows = true,
2849
+ fields = null,
2850
+ duals,
2851
+ coerceNumericStrings,
2852
+ onProgress,
2853
+ signal
1773
2854
  } = options;
1774
2855
  this._materialisesRows = materialisesRows;
1775
2856
  this._path = validatePath(filePath, allowedDir);
2857
+ this._duals = normaliseDuals(duals, this._path);
2858
+ this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
1776
2859
  this._memorySafetyFactor = memorySafetyFactor;
1777
2860
  this._symbolFilteringThreshold = symbolFilteringThreshold;
2861
+ if (onProgress !== void 0 && typeof onProgress !== "function") {
2862
+ throw new exports.QvdValidationError("onProgress must be a function", {
2863
+ provided: onProgress,
2864
+ type: typeof onProgress,
2865
+ file: this._path
2866
+ });
2867
+ }
2868
+ if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
2869
+ throw new exports.QvdValidationError("signal must be an AbortSignal", {
2870
+ provided: signal,
2871
+ type: typeof signal,
2872
+ file: this._path
2873
+ });
2874
+ }
2875
+ this._requestedFields = fields === void 0 ? null : fields;
2876
+ this._onProgress = onProgress;
2877
+ this._signal = signal;
1778
2878
  this._buffer = null;
1779
2879
  this._headerOffset = null;
1780
2880
  this._symbolTableOffset = null;
1781
2881
  this._indexTableOffset = null;
1782
2882
  this._header = null;
2883
+ this._allFields = null;
2884
+ this._selectedFields = null;
2885
+ this._fieldBitMetadataValidated = false;
1783
2886
  this._symbolTable = null;
1784
2887
  this._indexColumns = null;
1785
2888
  this._rowsDecoded = 0;
2889
+ this._bufferFirstRow = 0;
1786
2890
  this._fileSize = null;
1787
2891
  this._headerMatchesFile = false;
1788
2892
  }
2893
+ /**
2894
+ * Emits a progress event if a callback is registered.
2895
+ *
2896
+ * The same shape `QvdFileWriter._emitProgress` emits, deliberately: a caller who has written a
2897
+ * progress bar for a write should not have to write a second one for a read. The stage names
2898
+ * differ because the stages differ, but `symbol-table` and `index-table` mean the same thing on
2899
+ * both sides.
2900
+ *
2901
+ * @param {string} stage The current stage of the read.
2902
+ * @param {number} current The current progress value.
2903
+ * @param {number} total The total progress value.
2904
+ * @private
2905
+ */
2906
+ _emitProgress(stage, current, total) {
2907
+ if (this._onProgress) {
2908
+ const percent = total > 0 ? Math.round(current / total * 100) : 100;
2909
+ this._onProgress({ stage, current, total, percent });
2910
+ }
2911
+ }
2912
+ /**
2913
+ * Throws if the caller has cancelled the read.
2914
+ *
2915
+ * Throws `signal.reason` - a `DOMException` named `AbortError` unless the caller aborted with a
2916
+ * reason of their own. That is what `AbortSignal` means everywhere else in Node, and inventing
2917
+ * a `QvdAbortError` here would make this library's cancellation the one a caller has to special
2918
+ * case.
2919
+ *
2920
+ * @private
2921
+ */
2922
+ _throwIfAborted() {
2923
+ if (this._signal) {
2924
+ this._signal.throwIfAborted();
2925
+ }
2926
+ }
1789
2927
  /**
1790
2928
  * Reads the binary data of the QVD file.
1791
2929
  *
1792
- * LAZY LOADING OPTIMIZATION: When maxRows is specified, this method implements
1793
- * true lazy loading by reading only the necessary portions of the file from disk.
2930
+ * A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
2931
+ * needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
2932
+ * over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
1794
2933
  *
1795
- * For large files (e.g., 5GB), loading only the first 1000 rows can save significant
1796
- * memory and time:
1797
- * - Full load: 5GB in memory, ~30-60s load time
1798
- * - Lazy load (maxRows=1000): ~1.75-2GB in memory, ~2-5s load time
2934
+ * The saving is in the index table and the rows, not in the symbol table, which is read in
2935
+ * full whatever the window because a stored index in any row can address any symbol. So the
2936
+ * gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
2937
+ * values there is very little to save, which is what `symbolFilteringThreshold` and the
2938
+ * two-pass path exist for.
1799
2939
  *
1800
- * Algorithm for Lazy Loading:
2940
+ * Algorithm for a windowed read:
1801
2941
  * 1. Stream-read the file until XML header delimiter is found
1802
2942
  * 2. Parse header to determine symbol table and index table locations
1803
2943
  * 3. Calculate bytes needed: header + full symbol table + partial index table
@@ -1810,14 +2950,23 @@ var init_QvdFileReader = __esm({
1810
2950
  * - Streaming for header finding is efficient for unknown header sizes
1811
2951
  * - Direct byte-range reading for remaining data is fastest
1812
2952
  *
1813
- * @param {number|null} maxRows The maximum number of rows to load. If null, all data is loaded.
2953
+ * A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
2954
+ * table from the front of the file, and the window's records from wherever they sit. The bytes
2955
+ * between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
2956
+ * fixture a 0.4MB read rather than a 38MB one.
2957
+ *
2958
+ * @param {QvdRowWindow} window The rows to read.
1814
2959
  * @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
1815
2960
  * symbol and index tables on disk. This is the metadata-only path: the header is a few
1816
2961
  * kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
1817
2962
  * for a 40GB one.
2963
+ * @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
2964
+ * that is fewer than the window covers - see `_prepare`.
1818
2965
  * @private
1819
2966
  */
1820
- async _readData(maxRows = null, headerOnly = false) {
2967
+ async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
2968
+ this._throwIfAborted();
2969
+ this._emitProgress("read", 0, 1);
1821
2970
  const HEADER_DELIMITER = "\r\n\0";
1822
2971
  const CHUNK_SIZE = 64 * 1024;
1823
2972
  const stream = fs__default.default.createReadStream(this._path, {
@@ -1859,6 +3008,8 @@ var init_QvdFileReader = __esm({
1859
3008
  if (!isExpectedEarlyClose) {
1860
3009
  throw error;
1861
3010
  }
3011
+ } finally {
3012
+ await closeReadStream(stream);
1862
3013
  }
1863
3014
  if (headerDelimiterIndex === -1) {
1864
3015
  throw new exports.QvdCorruptedError(
@@ -1887,13 +3038,14 @@ var init_QvdFileReader = __esm({
1887
3038
  const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
1888
3039
  if (headerOnly) {
1889
3040
  this._buffer = headerBuffer.subarray(0, headerEndIndex);
3041
+ this._emitProgress("read", 1, 1);
1890
3042
  return;
1891
3043
  }
1892
3044
  let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
1893
3045
  if (headerFields && !Array.isArray(headerFields)) {
1894
3046
  headerFields = [headerFields];
1895
3047
  }
1896
- const columnCount = Array.isArray(headerFields) ? headerFields.length : 0;
3048
+ const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
1897
3049
  const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
1898
3050
  (value) => Number.isSafeInteger(value) && value >= 0
1899
3051
  );
@@ -1902,23 +3054,28 @@ var init_QvdFileReader = __esm({
1902
3054
  this._fileSize = fileSize;
1903
3055
  this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
1904
3056
  }
3057
+ const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
3058
+ const windowRows = resolved.limit;
1905
3059
  if (headerNumbersUsable && this._headerMatchesFile) {
1906
3060
  validateMemoryAvailability(
1907
3061
  symbolTableLength,
1908
- maxRows,
3062
+ windowRows,
1909
3063
  totalRows,
1910
3064
  this._path,
1911
3065
  this._memorySafetyFactor,
1912
3066
  columnCount,
1913
- this._materialisesRows
3067
+ this._materialisesRows,
3068
+ liveRows
1914
3069
  );
1915
3070
  }
1916
- if (maxRows === null) {
3071
+ if (window.offset === 0 && window.limit === null) {
1917
3072
  this._buffer = await fs__default.default.promises.readFile(this._path);
1918
3073
  this._fileSize = this._buffer.length;
3074
+ this._bufferFirstRow = 0;
3075
+ this._emitProgress("read", 1, 1);
1919
3076
  return;
1920
3077
  }
1921
- const rowsToLoad = Math.min(maxRows, totalRows);
3078
+ const rowsToLoad = windowRows;
1922
3079
  validateSymbolTableSizeEarly(symbolTableLength, this._path);
1923
3080
  for (const [name, value] of [
1924
3081
  ["Offset", symbolTableLength],
@@ -1934,39 +3091,78 @@ var init_QvdFileReader = __esm({
1934
3091
  });
1935
3092
  }
1936
3093
  }
3094
+ const skippedIndexBytes = resolved.offset * recordSize;
1937
3095
  const indexTableBytesToRead = rowsToLoad * recordSize;
1938
3096
  const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
3097
+ const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
1939
3098
  const fd = await fs__default.default.promises.open(this._path, "r");
1940
3099
  try {
1941
3100
  const { size: fileSize } = await fd.stat();
1942
3101
  this._fileSize = fileSize;
1943
- if (totalBytesToRead > fileSize) {
3102
+ if (fileBytesRequired > fileSize) {
1944
3103
  throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
1945
3104
  file: this._path,
1946
3105
  fileSize,
1947
- requiredBytes: totalBytesToRead,
3106
+ requiredBytes: fileBytesRequired,
1948
3107
  stage: "readData"
1949
3108
  });
1950
3109
  }
1951
3110
  this._buffer = Buffer.alloc(totalBytesToRead);
1952
- let position = 0;
1953
- while (position < totalBytesToRead) {
1954
- const length = Math.min(READ_CHUNK_SIZE, totalBytesToRead - position);
1955
- const { bytesRead } = await fd.read(this._buffer, position, length, position);
1956
- if (bytesRead === 0) {
1957
- throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
1958
- file: this._path,
1959
- fileSize,
1960
- bytesRead: position,
1961
- requiredBytes: totalBytesToRead,
1962
- stage: "readData"
1963
- });
1964
- }
1965
- position += bytesRead;
3111
+ await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
3112
+ if (indexTableBytesToRead > 0) {
3113
+ await this._readRange(
3114
+ fd,
3115
+ indexTableOffset,
3116
+ indexTableBytesToRead,
3117
+ indexTableOffset + skippedIndexBytes,
3118
+ fileSize,
3119
+ fileBytesRequired
3120
+ );
1966
3121
  }
3122
+ this._bufferFirstRow = resolved.offset;
1967
3123
  } finally {
1968
3124
  await fd.close();
1969
3125
  }
3126
+ this._emitProgress("read", 1, 1);
3127
+ }
3128
+ /**
3129
+ * Reads one byte range of the file into the buffer.
3130
+ *
3131
+ * Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
3132
+ * 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
3133
+ * try/catch can intercept.
3134
+ *
3135
+ * @param {import('fs/promises').FileHandle} fd The open file.
3136
+ * @param {number} bufferOffset Where in the buffer to write.
3137
+ * @param {number} byteCount How many bytes to read.
3138
+ * @param {number} filePosition Where in the file to read from.
3139
+ * @param {number} fileSize The file's size, for the error.
3140
+ * @param {number} requiredBytes Bytes the whole read needs, for the error.
3141
+ * @private
3142
+ */
3143
+ async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
3144
+ assert2__default.default(this._buffer, "The read buffer has not been allocated.");
3145
+ let done = 0;
3146
+ while (done < byteCount) {
3147
+ const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
3148
+ const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
3149
+ if (bytesRead === 0) {
3150
+ throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
3151
+ file: this._path,
3152
+ fileSize,
3153
+ // Two numbers, because they stopped being the same one when a window began reading two
3154
+ // ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
3155
+ // file it gave up. Reporting the position under the name of the count made a windowed
3156
+ // read of a truncated file claim tens of megabytes had been read when a few hundred
3157
+ // bytes had.
3158
+ bytesRead: done,
3159
+ filePosition: filePosition + done,
3160
+ requiredBytes,
3161
+ stage: "readData"
3162
+ });
3163
+ }
3164
+ done += bytesRead;
3165
+ }
1970
3166
  }
1971
3167
  /**
1972
3168
  * Parses the XML header of the QVD file. This method is part of the parsing process
@@ -2026,6 +3222,8 @@ var init_QvdFileReader = __esm({
2026
3222
  this._headerOffset = headerBeginIndex;
2027
3223
  this._symbolTableOffset = headerEndIndex;
2028
3224
  this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3225
+ this._allFields = fieldList;
3226
+ this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
2029
3227
  }
2030
3228
  /**
2031
3229
  * Establishes the geometry of the index table, and validates it.
@@ -2036,14 +3234,15 @@ var init_QvdFileReader = __esm({
2036
3234
  * about keeping the sign in step with the other one: the two could drift, and #113 is what
2037
3235
  * that looks like when they do. There is one copy now.
2038
3236
  *
2039
- * @param {number|null} rowLimit Maximum rows of interest, or null for all of them.
3237
+ * @param {QvdRowWindow} window The rows of interest, as file row indices.
2040
3238
  * @param {string} stage Stage name for any error raised here.
2041
3239
  * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
2042
- * indexBuffer: Buffer}} The record geometry.
3240
+ * indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
3241
+ * record, so the decoder always counts from zero.
2043
3242
  * @private
2044
3243
  */
2045
- _planIndexTable(rowLimit, stage) {
2046
- if (!this._buffer || !this._header || !this._indexTableOffset) {
3244
+ _planIndexTable(window, stage) {
3245
+ if (!this._buffer || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
2047
3246
  throw new exports.QvdCorruptedError(
2048
3247
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
2049
3248
  {
@@ -2052,14 +3251,12 @@ var init_QvdFileReader = __esm({
2052
3251
  }
2053
3252
  );
2054
3253
  }
2055
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
2056
- if (!Array.isArray(fields)) {
2057
- fields = [fields];
2058
- }
3254
+ const allFields = this._allFields;
3255
+ const fields = this._selectedFields;
2059
3256
  const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
2060
3257
  const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
2061
- const rowsToLoad = rowLimit !== null ? Math.min(rowLimit, totalRows) : totalRows;
2062
3258
  const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
3259
+ const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
2063
3260
  validateIndexTableMetadata(
2064
3261
  recordSize,
2065
3262
  totalRows,
@@ -2068,11 +3265,20 @@ var init_QvdFileReader = __esm({
2068
3265
  this._buffer.length,
2069
3266
  rowsToLoad,
2070
3267
  this._path,
2071
- this._fileSize
3268
+ this._fileSize,
3269
+ firstRow,
3270
+ this._bufferFirstRow
2072
3271
  );
2073
- const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
2074
- for (const field of fields) {
2075
- validateFieldBitMetadata(field, recordSize, this._path);
3272
+ const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
3273
+ const indexBuffer = this._buffer.subarray(
3274
+ this._indexTableOffset + bufferRecordStart,
3275
+ this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
3276
+ );
3277
+ if (!this._fieldBitMetadataValidated) {
3278
+ for (const field of allFields) {
3279
+ validateFieldBitMetadata(field, recordSize, this._path);
3280
+ }
3281
+ this._fieldBitMetadataValidated = true;
2076
3282
  }
2077
3283
  assert2__default.default(
2078
3284
  rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
@@ -2084,31 +3290,45 @@ var init_QvdFileReader = __esm({
2084
3290
  * Analyzes the index table to determine which symbols are actually needed.
2085
3291
  * This is used for two-pass symbol filtering optimization.
2086
3292
  *
2087
- * @param {number} maxRows The maximum number of rows to analyze.
2088
- * @return {Promise<Map<string, Set<number>>>} Map of field names to Set of needed symbol indices.
3293
+ * Only the selected fields are analysed. An unselected field's symbols are never parsed, so
3294
+ * there is nothing for a usage set to filter and decoding its column would be a pass over the
3295
+ * whole window for an answer nobody reads.
3296
+ *
3297
+ * @param {QvdRowWindow} window The rows to analyse.
3298
+ * @return {Promise<Array<Set<number>>>} One set of needed symbol indices per selected field, in
3299
+ * the same order `_parseSymbolTable` walks them.
2089
3300
  * @private
2090
3301
  */
2091
- async _analyzeIndexTableSymbolUsage(maxRows) {
2092
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "analyzeIndexTableSymbolUsage");
2093
- const symbolUsage = /* @__PURE__ */ new Map();
2094
- const column = new Int32Array(rowsToLoad);
2095
- fields.forEach((field) => {
3302
+ async _analyzeIndexTableSymbolUsage(window) {
3303
+ const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
3304
+ const symbolUsage = [];
3305
+ const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
3306
+ const column = new Int32Array(sliceRows);
3307
+ fields.forEach((field, position) => {
3308
+ this._throwIfAborted();
2096
3309
  const needed = /* @__PURE__ */ new Set();
2097
- symbolUsage.set(field["FieldName"], needed);
2098
- decodeIndexColumn(
2099
- indexBuffer,
2100
- recordSize,
2101
- rowsToLoad,
2102
- parseInt(field["BitOffset"], 10),
2103
- parseInt(field["BitWidth"], 10),
2104
- parseInt(field["Bias"], 10),
2105
- column
2106
- );
2107
- for (let row = 0; row < rowsToLoad; row++) {
2108
- if (column[row] >= 0) {
2109
- needed.add(column[row]);
3310
+ symbolUsage[position] = needed;
3311
+ const bitOffset = parseInt(field["BitOffset"], 10);
3312
+ const bitWidth = parseInt(field["BitWidth"], 10);
3313
+ const bias = parseInt(field["Bias"], 10);
3314
+ for (let first = 0; first < rowsToLoad; first += sliceRows) {
3315
+ const count = Math.min(sliceRows, rowsToLoad - first);
3316
+ decodeIndexColumn(
3317
+ first === 0 ? indexBuffer : indexBuffer.subarray(first * recordSize),
3318
+ recordSize,
3319
+ count,
3320
+ bitOffset,
3321
+ bitWidth,
3322
+ bias,
3323
+ column
3324
+ );
3325
+ for (let row = 0; row < count; row++) {
3326
+ if (column[row] >= 0) {
3327
+ needed.add(column[row]);
3328
+ }
2110
3329
  }
2111
3330
  }
3331
+ this._emitProgress("symbol-analysis", position + 1, fields.length);
2112
3332
  });
2113
3333
  return symbolUsage;
2114
3334
  }
@@ -2116,12 +3336,20 @@ var init_QvdFileReader = __esm({
2116
3336
  * Parses the symbol table of the QVD file. This method is part of the parsing process
2117
3337
  * and should not be called directly.
2118
3338
  *
2119
- * @param {Map<string, Set<number>>|null} symbolsToKeep Optional map of field names to symbol indices to keep.
2120
- * If provided, only these symbols will be parsed (two-pass filtering optimization).
2121
- * @param {number|null} maxRows Optional maximum number of rows being loaded (for memory estimation).
3339
+ * A field the caller did not select is skipped whole. Its symbol area is neither scanned nor
3340
+ * parsed - the per-field `Offset` and `Length` say exactly where it is, so there is nothing to
3341
+ * walk past - and that is where field selection earns its keep. The index decode is cheap by
3342
+ * comparison; parsing symbols is not.
3343
+ *
3344
+ * @param {Array<Set<number>>|null} symbolsToKeep Optional set of symbol indices to keep per
3345
+ * selected field, indexed by position. If provided, only these symbols will be parsed
3346
+ * (two-pass filtering optimization).
3347
+ * @param {number} rowsToLoad Rows the read covers, for memory estimation.
3348
+ * @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
3349
+ * that is fewer than the window covers - see `_prepare`.
2122
3350
  */
2123
- async _parseSymbolTable(symbolsToKeep = null, maxRows = null) {
2124
- if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset) {
3351
+ async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
3352
+ if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
2125
3353
  throw new exports.QvdCorruptedError(
2126
3354
  "The QVD file has not been loaded in the proper order or has not been loaded at all.",
2127
3355
  {
@@ -2130,7 +3358,8 @@ var init_QvdFileReader = __esm({
2130
3358
  }
2131
3359
  );
2132
3360
  }
2133
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
3361
+ const allFields = this._allFields;
3362
+ const fields = this._selectedFields;
2134
3363
  const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
2135
3364
  const symbolTableSize = symbolBuffer.length;
2136
3365
  const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
@@ -2138,51 +3367,34 @@ var init_QvdFileReader = __esm({
2138
3367
  if (this._headerMatchesFile) {
2139
3368
  validateMemoryAvailability(
2140
3369
  symbolTableSize,
2141
- maxRows,
3370
+ rowsToLoad,
2142
3371
  totalRows,
2143
3372
  this._path,
2144
3373
  this._memorySafetyFactor,
2145
- Array.isArray(fields) ? fields.length : 1,
2146
- this._materialisesRows
3374
+ fields.length,
3375
+ this._materialisesRows,
3376
+ liveRows
2147
3377
  );
2148
3378
  }
2149
- warnLargeSymbolTable(
2150
- symbolTableSize,
2151
- maxRows,
2152
- totalRows,
2153
- Array.isArray(fields) ? fields.length : 1,
2154
- this._materialisesRows
2155
- );
2156
- if (!Array.isArray(fields)) {
2157
- fields = [fields];
2158
- }
2159
- for (const field of fields) {
3379
+ warnLargeSymbolTable(symbolTableSize, rowsToLoad, totalRows, fields.length, this._materialisesRows);
3380
+ for (const field of allFields) {
2160
3381
  validateFieldMetadata(field, symbolBuffer.length, this._path);
2161
3382
  }
2162
- this._symbolTable = fields.map((field) => {
3383
+ this._symbolTable = fields.map((field, position) => {
3384
+ this._throwIfAborted();
2163
3385
  const symbolsOffset = parseInt(field["Offset"], 10);
2164
3386
  const symbolsLength = parseInt(field["Length"], 10);
2165
- const fieldName = field["FieldName"];
2166
- const neededSymbols = symbolsToKeep ? symbolsToKeep.get(fieldName) : null;
2167
- const filteringEnabled = neededSymbols !== null;
2168
- const symbols = [];
2169
- let symbolIndex = 0;
2170
- for (let pointer = symbolsOffset; pointer < symbolsOffset + symbolsLength; pointer++) {
2171
- const typeByte = symbolBuffer[pointer++];
2172
- const shouldKeepSymbol = !filteringEnabled || !!(neededSymbols && neededSymbols.has(symbolIndex));
2173
- const { symbol, bytesRead } = parseSymbol(
2174
- typeByte,
2175
- symbolBuffer,
2176
- pointer,
2177
- symbolBuffer.length,
2178
- fieldName,
2179
- this._path,
2180
- shouldKeepSymbol
2181
- );
2182
- symbols.push(symbol);
2183
- pointer += bytesRead - 1;
2184
- symbolIndex++;
2185
- }
3387
+ const symbols = parseFieldSymbols(
3388
+ symbolBuffer,
3389
+ symbolsOffset,
3390
+ symbolsOffset + symbolsLength,
3391
+ // By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
3392
+ // `this._selectedFields`, so position is the one key that cannot collide.
3393
+ symbolsToKeep ? symbolsToKeep[position] : null,
3394
+ field["FieldName"],
3395
+ this._path
3396
+ );
3397
+ this._emitProgress("symbol-table", position + 1, fields.length);
2186
3398
  return symbols;
2187
3399
  });
2188
3400
  }
@@ -2201,13 +3413,18 @@ var init_QvdFileReader = __esm({
2201
3413
  * same for every row, so they are hoisted out of the loop and the inner loop does arithmetic
2202
3414
  * into a typed array and nothing else. Rows are assembled later, once, in `load()`.
2203
3415
  *
2204
- * @param {number|null} maxRows The maximum number of rows to parse. If null, all rows are parsed.
3416
+ * The window is what makes chunked iteration cheap: `decodeIndexColumn` walks records by
3417
+ * `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
3418
+ * and how many iterations run. Nothing about the decoder changed to support it.
3419
+ *
3420
+ * @param {QvdRowWindow} window The rows to decode.
2205
3421
  */
2206
- async _parseIndexTable(maxRows = null) {
2207
- const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "parseIndexTable");
3422
+ async _parseIndexTable(window) {
3423
+ const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
2208
3424
  this._rowsDecoded = rowsToLoad;
2209
- this._indexColumns = fields.map(
2210
- (field) => decodeIndexColumn(
3425
+ this._indexColumns = fields.map((field, position) => {
3426
+ this._throwIfAborted();
3427
+ const column = decodeIndexColumn(
2211
3428
  indexBuffer,
2212
3429
  recordSize,
2213
3430
  rowsToLoad,
@@ -2215,8 +3432,10 @@ var init_QvdFileReader = __esm({
2215
3432
  parseInt(field["BitWidth"], 10),
2216
3433
  parseInt(field["Bias"], 10),
2217
3434
  new Int32Array(rowsToLoad)
2218
- )
2219
- );
3435
+ );
3436
+ this._emitProgress("index-table", position + 1, fields.length);
3437
+ return column;
3438
+ });
2220
3439
  }
2221
3440
  /**
2222
3441
  * Reads the file's schema and header metadata, without touching the symbol or index tables.
@@ -2233,8 +3452,11 @@ var init_QvdFileReader = __esm({
2233
3452
  * @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
2234
3453
  */
2235
3454
  async loadMetadata() {
2236
- await this._readData(null, true);
3455
+ await this._readData({ offset: 0, limit: null }, true);
3456
+ this._emitProgress("header", 0, 1);
2237
3457
  await this._parseHeader();
3458
+ this._emitProgress("header", 1, 1);
3459
+ this._throwIfAborted();
2238
3460
  assert2__default.default(this._header, "The QVD file header has not been parsed.");
2239
3461
  const header = this._header["QvdTableHeader"];
2240
3462
  let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
@@ -2269,137 +3491,337 @@ var init_QvdFileReader = __esm({
2269
3491
  /**
2270
3492
  * Loads the QVD file into memory and parses it.
2271
3493
  *
2272
- * @param {number|null} maxRows The maximum number of rows to load. If null, all rows are loaded.
2273
- * Must be a non-negative integer when given.
2274
- * @throws {QvdValidationError} If maxRows is neither null nor a non-negative integer.
3494
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
3495
+ * The rows to load. A number or null means what it always meant - the first N rows, or all of
3496
+ * them - and `{offset, limit}` is the same thing said more precisely, so `5` and
3497
+ * `{offset: 0, limit: 5}` are one read. `maxRows` is accepted as a second name for `limit`.
3498
+ * @throws {QvdValidationError} If the window is not a non-negative integer, null, or a valid
3499
+ * `{offset, limit}` object.
2275
3500
  * @return {Promise<QvdDataFrame>} The loaded QVD file.
2276
3501
  */
2277
- async load(maxRows = null) {
2278
- const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
3502
+ async load(window = null) {
3503
+ const rows = normaliseWindow(window, this._path);
3504
+ const prepared = await this._prepare(rows);
3505
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
3506
+ const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
3507
+ return new exports.QvdDataFrame(
3508
+ data,
3509
+ prepared.columns,
3510
+ prepared.metadata,
3511
+ {
3512
+ ...prepared.loadStats,
3513
+ rowsLoaded: data.length
3514
+ },
3515
+ prepared.storedSymbols
3516
+ );
3517
+ }
3518
+ /**
3519
+ * Reads the file as columns, without ever materialising rows.
3520
+ *
3521
+ * Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
3522
+ * What it keeps instead is what the decoder already produced: one `Int32Array` of stored
3523
+ * indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
3524
+ * fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
3525
+ * bytes per row rather than a boxed value per cell, and the symbols are a few thousand
3526
+ * entries shared across every row that uses them.
3527
+ *
3528
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
3529
+ * The rows to decode, in the same spellings `load()` accepts.
3530
+ * @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
3531
+ */
3532
+ async loadColumnar(window = null) {
3533
+ const rows = normaliseWindow(window, this._path);
3534
+ const prepared = await this._prepare(rows, null, true);
3535
+ await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
3536
+ const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
2279
3537
  assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
2280
- const indexColumns = this._indexColumns;
2281
- const fieldCount = indexColumns.length;
2282
- const data = new Array(this._rowsDecoded);
2283
- for (let row = 0; row < this._rowsDecoded; row++) {
2284
- const values = new Array(fieldCount);
2285
- for (let field = 0; field < fieldCount; field++) {
2286
- const symbolIndex = indexColumns[field][row];
2287
- values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
2288
- }
2289
- data[row] = values;
2290
- }
2291
- loadStats.rowsLoaded = data.length;
2292
- return new exports.QvdDataFrame(data, columns, metadata, loadStats);
3538
+ return new QvdColumnTable2({
3539
+ columns: prepared.columns,
3540
+ codesByField: this._indexColumns,
3541
+ symbolsByField: prepared.resolvedByField,
3542
+ halvesByField: prepared.halvesByField,
3543
+ rowCount: this._rowsDecoded,
3544
+ metadata: prepared.metadata,
3545
+ storedSymbols: prepared.storedSymbols,
3546
+ loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
3547
+ });
2293
3548
  }
2294
3549
  /**
2295
- * Reads and decodes the file, stopping short of building rows.
3550
+ * Yields the window as data frames of at most `chunkSize` rows.
2296
3551
  *
2297
- * Everything `load()` and `loadColumnar()` have in common, which is everything except the
2298
- * shape of the answer. Two read paths for one binary format is the drift risk #113 is the
2299
- * standing example of - a stored index resolved one way here and another way there returns
2300
- * plausible wrong values and throws nothing - so there is one path, and the two entry points
2301
- * differ only in what they do with what it returns.
3552
+ * The file is opened, read and parsed **once**; only the index decode and the row building
3553
+ * happen per chunk. That is the whole reason this exists as a method rather than as a loop of
3554
+ * `load({offset, limit})` calls at the call site: the symbol table has to be parsed in full
3555
+ * whatever the chunk size - a stored index in the last chunk can address the first symbol -
3556
+ * and re-parsing it per chunk is what makes the obvious implementation cost more than a plain
3557
+ * load rather than less. PyQvd's chunked read does re-read it, and the comment on #140 records
3558
+ * that as a limitation rather than a design.
2302
3559
  *
2303
- * @param {number|null} maxRows Maximum rows to decode, or null for all of them.
2304
- * @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
2305
- * resolvedByField: Array<Array<any>>}>} The decoded file.
2306
- * @private
3560
+ * What it bounds is row materialisation, which is what actually dominates a large read's heap.
3561
+ * Two chunks of rows are alive at a time, not one - `for await` keeps the yielded frame
3562
+ * reachable while this generator builds the next - which is why `liveRows` below is
3563
+ * `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
3564
+ *
3565
+ * A window covering no rows yields nothing at all, rather than one empty frame - so
3566
+ * `for await` over an exhausted offset does nothing, which is what a paging loop wants.
3567
+ *
3568
+ * @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
3569
+ * The rows to cover, in the same spellings `load()` accepts.
3570
+ * @param {number} chunkSize Rows per frame. Must be a positive integer.
3571
+ * @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
2307
3572
  */
2308
- async _decode(maxRows = null) {
2309
- if (maxRows !== null && (typeof maxRows !== "number" || !Number.isInteger(maxRows) || maxRows < 0)) {
2310
- throw new exports.QvdValidationError("maxRows must be a non-negative integer, or null to load all rows", {
2311
- provided: maxRows,
2312
- type: typeof maxRows,
3573
+ async *iterateRows(window, chunkSize) {
3574
+ if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
3575
+ throw new exports.QvdValidationError("chunkSize must be a positive integer", {
3576
+ provided: chunkSize,
3577
+ type: typeof chunkSize,
2313
3578
  file: this._path
2314
3579
  });
2315
3580
  }
2316
- await this._readData(maxRows);
3581
+ const liveRows = { rows: chunkSize * 2, perChunk: 2 };
3582
+ const rows = normaliseWindow(window, this._path);
3583
+ const prepared = await this._prepare(rows, liveRows);
3584
+ for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
3585
+ this._throwIfAborted();
3586
+ const count = Math.min(chunkSize, prepared.rowsAvailable - done);
3587
+ const offset = prepared.offset + done;
3588
+ await this._parseIndexTable({ offset, limit: count });
3589
+ const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
3590
+ yield new exports.QvdDataFrame(
3591
+ data,
3592
+ prepared.columns,
3593
+ prepared.metadata,
3594
+ {
3595
+ ...prepared.loadStats,
3596
+ offset,
3597
+ rowsLoaded: data.length
3598
+ },
3599
+ prepared.storedSymbols
3600
+ );
3601
+ }
3602
+ }
3603
+ /**
3604
+ * Reads the file and resolves its symbols, stopping short of decoding any rows.
3605
+ *
3606
+ * Everything `load()`, `loadColumnar()` and `iterateRows()` have in common, which is everything
3607
+ * that depends on the file rather than on the window. Two read paths for one binary format is
3608
+ * the drift risk #113 is the standing example of - a stored index resolved one way here and
3609
+ * another way there returns plausible wrong values and throws nothing - so there is one path,
3610
+ * and the entry points differ only in what they do with what it returns and how many rows they
3611
+ * ask for at a time.
3612
+ *
3613
+ * @param {QvdRowWindow} window The rows the read covers.
3614
+ * @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
3615
+ * is fewer than the window covers, and how many of them one row of the caller's chunk size
3616
+ * accounts for. Only `iterateRows` passes it; every other read holds what it covers.
3617
+ * @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
3618
+ * cells do not show them, which only a columnar read has a use for.
3619
+ * @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
3620
+ * resolvedByField: Array<Array<any>>,
3621
+ * halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
3622
+ * storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
3623
+ * rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
3624
+ * against it.
3625
+ * @private
3626
+ */
3627
+ async _prepare(window, liveRows = null, wantHalves = false) {
3628
+ this._throwIfAborted();
3629
+ await this._readData(window, false, liveRows);
3630
+ this._emitProgress("header", 0, 1);
2317
3631
  await this._parseHeader();
3632
+ this._emitProgress("header", 1, 1);
3633
+ this._throwIfAborted();
3634
+ assert2__default.default(this._header, "The QVD file header has not been parsed.");
3635
+ const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
3636
+ const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3637
+ const resolved = resolveWindow(window, totalRows);
3638
+ const rowsAvailable = resolved.limit;
2318
3639
  let symbolsToKeep = null;
2319
3640
  let symbolsKept = null;
2320
- if (maxRows !== null && this._header) {
2321
- const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
3641
+ if (window.limit !== null || window.offset > 0) {
2322
3642
  if (symbolTableLength > this._symbolFilteringThreshold) {
2323
- symbolsToKeep = await this._analyzeIndexTableSymbolUsage(maxRows);
2324
- symbolsKept = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
3643
+ symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
3644
+ symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
2325
3645
  }
2326
3646
  }
2327
- await this._parseSymbolTable(symbolsToKeep, maxRows);
2328
- await this._parseIndexTable(maxRows);
2329
- assert2__default.default(this._header, "The QVD file header has not been parsed.");
3647
+ await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
2330
3648
  assert2__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
2331
- assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
2332
- const resolvedByField = this._symbolTable.map((symbols) => {
2333
- const resolved = new Array(symbols.length);
2334
- for (let index = 0; index < symbols.length; index++) {
2335
- const value = symbols[index]?.toPrimaryValue();
2336
- resolved[index] = typeof value === "string" && value.trim() !== "" && !isNaN(Number(value)) ? Number(value) : value;
3649
+ this._throwIfAborted();
3650
+ assert2__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
3651
+ const resolvedByField = [];
3652
+ const halvesByField = [];
3653
+ const entries = [];
3654
+ this._symbolTable.forEach((symbols, position) => {
3655
+ const { values, entry, halves } = resolveFieldSymbols(
3656
+ symbols,
3657
+ // @ts-ignore - asserted above
3658
+ this._selectedFields[position]["FieldName"],
3659
+ this._duals,
3660
+ this._coerceNumericStrings,
3661
+ wantHalves
3662
+ );
3663
+ resolvedByField.push(values);
3664
+ halvesByField.push(halves);
3665
+ if (entry !== null) {
3666
+ entries.push(entry);
2337
3667
  }
2338
- return resolved;
2339
3668
  });
2340
- let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
2341
- if (!Array.isArray(fields)) {
2342
- fields = [fields];
2343
- }
2344
- const columns = fields.map((field) => field["FieldName"]);
3669
+ const columns = this._selectedFields.map((field) => field["FieldName"]);
2345
3670
  const metadata = this._header["QvdTableHeader"];
3671
+ const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
3672
+ if (storedSymbols !== null) {
3673
+ attachStoredSymbols(metadata, storedSymbols);
3674
+ }
2346
3675
  const loadStats = {
2347
- symbolTableBytes: parseInt(this._header["QvdTableHeader"]["Offset"], 10),
2348
- totalRows: parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10),
2349
- rowsLoaded: this._rowsDecoded,
3676
+ symbolTableBytes: symbolTableLength,
3677
+ totalRows,
3678
+ rowsLoaded: 0,
3679
+ offset: resolved.offset,
2350
3680
  symbolFiltering: symbolsToKeep !== null,
2351
3681
  symbolsKept
2352
3682
  };
2353
- return { columns, metadata, loadStats, resolvedByField };
3683
+ return {
3684
+ columns,
3685
+ metadata,
3686
+ loadStats,
3687
+ resolvedByField,
3688
+ halvesByField,
3689
+ storedSymbols,
3690
+ rowsAvailable,
3691
+ offset: resolved.offset
3692
+ };
2354
3693
  }
2355
3694
  /**
2356
- * Reads the file as columns, without ever materialising rows.
3695
+ * Builds rows from the columns currently decoded.
2357
3696
  *
2358
- * Shares every step with `load()` up to the point where rows would be built - see `_decode`.
2359
- * What it keeps instead is what the decoder already produced: one `Int32Array` of stored
2360
- * indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
2361
- * fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
2362
- * bytes per row rather than a boxed value per cell, and the symbols are a few thousand
2363
- * entries shared across every row that uses them.
3697
+ * `data` stays eager: of the four ways this library is used - a full read, a preview already
3698
+ * bounded by a limit, writing an array out, and reading metadata - not one is helped by
3699
+ * materialising a row only when it is touched, and a lazy accessor would cost a proxy, a cache
3700
+ * and mutation semantics to serve none of them. A caller who wants columns without paying for
3701
+ * rows uses `QvdColumnTable`, which stops before this loop.
2364
3702
  *
2365
- * @param {number|null} maxRows The maximum number of rows to decode.
2366
- * @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
3703
+ * @param {Array<Array<any>>} resolvedByField One resolved value per distinct symbol, per field.
3704
+ * @param {number} progressBase Rows already delivered before this call, so that progress over a
3705
+ * chunked iteration counts the whole window rather than restarting at every chunk.
3706
+ * @param {number} progressTotal Rows the whole window covers.
3707
+ * @return {Array<Array<any>>} The rows.
3708
+ * @private
2367
3709
  */
2368
- async loadColumnar(maxRows = null) {
2369
- const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
2370
- const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
3710
+ _buildRows(resolvedByField, progressBase, progressTotal) {
2371
3711
  assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
2372
- return new QvdColumnTable2({
2373
- columns,
2374
- codesByField: this._indexColumns,
2375
- symbolsByField: resolvedByField,
2376
- rowCount: this._rowsDecoded,
2377
- metadata,
2378
- loadStats
2379
- });
3712
+ const indexColumns = this._indexColumns;
3713
+ const fieldCount = indexColumns.length;
3714
+ const rowCount = this._rowsDecoded;
3715
+ const data = new Array(rowCount);
3716
+ const reportInterval = Math.max(1, Math.floor(progressTotal / 100));
3717
+ for (let row = 0; row < rowCount; row++) {
3718
+ const values = new Array(fieldCount);
3719
+ for (let field = 0; field < fieldCount; field++) {
3720
+ const symbolIndex = indexColumns[field][row];
3721
+ values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
3722
+ }
3723
+ data[row] = values;
3724
+ if ((progressBase + row + 1) % reportInterval === 0 || row + 1 === rowCount) {
3725
+ this._throwIfAborted();
3726
+ this._emitProgress("rows", progressBase + row + 1, progressTotal);
3727
+ }
3728
+ }
3729
+ return data;
2380
3730
  }
2381
3731
  };
2382
3732
  }
2383
3733
  });
2384
3734
 
2385
3735
  // src/QvdDataFrame.js
3736
+ function defaultFieldHeader(fieldName) {
3737
+ return {
3738
+ FieldName: fieldName,
3739
+ BitOffset: 0,
3740
+ BitWidth: 0,
3741
+ Bias: 0,
3742
+ NoOfSymbols: 0,
3743
+ Offset: 0,
3744
+ Length: 0,
3745
+ Comment: "",
3746
+ NumberFormat: {
3747
+ Type: "UNKNOWN",
3748
+ nDec: "0",
3749
+ UseThou: "0",
3750
+ Fmt: "",
3751
+ Dec: "",
3752
+ Thou: ""
3753
+ },
3754
+ Tags: {}
3755
+ };
3756
+ }
3757
+ function defaultHeader(columns) {
3758
+ return {
3759
+ QvBuildNo: 50667,
3760
+ CreatorDoc: "",
3761
+ CreateUtcTime: "",
3762
+ SourceCreateUtcTime: "",
3763
+ SourceFileUtcTime: "",
3764
+ SourceFileSize: -1,
3765
+ StaleUtcTime: "",
3766
+ TableName: "",
3767
+ Fields: {
3768
+ QvdFieldHeader: columns.map(defaultFieldHeader)
3769
+ },
3770
+ NoOfRecords: 0,
3771
+ RecordByteSize: 0,
3772
+ Offset: 0,
3773
+ Length: 0,
3774
+ Compression: "",
3775
+ Comment: "",
3776
+ EncryptionInfo: "",
3777
+ TableTags: "",
3778
+ ProfilingData: "",
3779
+ Lineage: {}
3780
+ };
3781
+ }
2386
3782
  exports.QvdDataFrame = void 0;
2387
3783
  var init_QvdDataFrame = __esm({
2388
3784
  "src/QvdDataFrame.js"() {
2389
3785
  init_QvdErrors();
3786
+ init_cellRules();
3787
+ init_readOptions();
3788
+ init_storedSymbols();
3789
+ __name(defaultFieldHeader, "defaultFieldHeader");
3790
+ __name(defaultHeader, "defaultHeader");
2390
3791
  exports.QvdDataFrame = class _QvdDataFrame {
3792
+ static {
3793
+ __name(this, "QvdDataFrame");
3794
+ }
2391
3795
  /**
2392
3796
  * Represents the data frame stored inside a QVD file.
3797
+ *
3798
+ * The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
3799
+ * its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
3800
+ * Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
3801
+ * holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
3802
+ * header's record did for a frame built from some of a read's columns.
3803
+ *
2393
3804
  * @param {Array<Array<any>>} data The data of the data frame.
2394
3805
  * @param {Array<string>} columns The columns of the data frame.
2395
3806
  * @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
2396
3807
  * @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
3808
+ * @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
3809
+ * shows only one half of its symbol (optional) - see `storedSymbols`.
3810
+ * @throws {QvdValidationError} If the record is malformed.
2397
3811
  */
2398
- constructor(data, columns, metadata = null, loadStats = null) {
3812
+ constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
2399
3813
  this._data = data;
2400
3814
  this._columns = columns;
2401
3815
  this._metadata = metadata;
3816
+ this._ownsMetadata = false;
2402
3817
  this._loadStats = loadStats;
3818
+ this._storedSymbols = narrowStoredSymbols(
3819
+ normaliseStoredSymbols(
3820
+ // @ts-ignore - a symbol-keyed property the reader defines on the header object
3821
+ storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
3822
+ ),
3823
+ columns
3824
+ );
2403
3825
  }
2404
3826
  /**
2405
3827
  * Returns the data of the data frame.
@@ -2426,15 +3848,40 @@ var init_QvdDataFrame = __esm({
2426
3848
  get metadata() {
2427
3849
  return this._metadata;
2428
3850
  }
3851
+ /**
3852
+ * What the frame's cells were read from, where a cell shows only one half of its symbol.
3853
+ *
3854
+ * A dual read as its number has a text the cell does not show; one read as its text has a number;
3855
+ * a string read as a number has the text it was spelled with. The record keeps those halves, per
3856
+ * field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
3857
+ * and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
3858
+ * `select`, `toDict` and `fromDict`.
3859
+ *
3860
+ * Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
3861
+ * the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
3862
+ * pure number.
3863
+ *
3864
+ * @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
3865
+ * showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
3866
+ * in the record it was given or found - one from `select`, or one built from some of a read's
3867
+ * columns and its header - has an empty record rather than null, because a frame given null takes
3868
+ * the record its header carries, which describes every field of the read.
3869
+ */
3870
+ get storedSymbols() {
3871
+ return this._storedSymbols;
3872
+ }
2429
3873
  /**
2430
3874
  * Returns statistics about the read that produced this data frame.
2431
3875
  *
2432
- * Only a frame returned by fromQvd() carries these; fromDict(), head() and tail() produce
2433
- * frames that describe no particular read, and report null rather than a stale figure.
3876
+ * Carried by every frame that came from a file - `fromQvd()`, and each chunk `iterate()` yields,
3877
+ * which is how a chunk reports its `offset`. `fromDict()`, `head()`, `tail()`, `rows()` and
3878
+ * `select()` describe no particular read and report null rather than a stale figure.
2434
3879
  *
2435
3880
  * The main use is confirming that a lazy load actually filtered the symbol table:
2436
3881
  * `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
2437
- * survived it, which for a small maxRows should be a tiny fraction of the file's total.
3882
+ * survived it. Note that a bounded read does not filter on its own - the two-pass path engages
3883
+ * only above `symbolFilteringThreshold`, so on a file below it this reports false and every
3884
+ * symbol was parsed however few rows were asked for.
2438
3885
  *
2439
3886
  * @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
2440
3887
  */
@@ -2543,54 +3990,40 @@ var init_QvdDataFrame = __esm({
2543
3990
  * @property {string} [profilingData] - Profiling data
2544
3991
  * @property {Object|string} [lineage] - Lineage
2545
3992
  */
3993
+ /**
3994
+ * The header a metadata setter may change: this frame's own.
3995
+ *
3996
+ * A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
3997
+ * `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
3998
+ * change copies it, and a change made through one frame never reaches another. A frame with no header
3999
+ * gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
4000
+ * so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
4001
+ *
4002
+ * @return {any} The header.
4003
+ */
4004
+ _ownMetadata() {
4005
+ if (!this._metadata) {
4006
+ this._metadata = defaultHeader(this._columns);
4007
+ } else if (!this._ownsMetadata) {
4008
+ const record = this._metadata[STORED_SYMBOLS];
4009
+ this._metadata = structuredClone(this._metadata);
4010
+ if (record) {
4011
+ attachStoredSymbols(this._metadata, record);
4012
+ }
4013
+ }
4014
+ this._ownsMetadata = true;
4015
+ return this._metadata;
4016
+ }
2546
4017
  /**
2547
4018
  * Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
4019
+ *
4020
+ * The change applies to this frame only, never to a frame it was derived from or shares a header
4021
+ * with.
4022
+ *
2548
4023
  * @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
2549
4024
  */
2550
4025
  setFileMetadata(metadata) {
2551
- if (!this._metadata) {
2552
- this._metadata = {
2553
- QvBuildNo: 50667,
2554
- CreatorDoc: "",
2555
- CreateUtcTime: "",
2556
- SourceCreateUtcTime: "",
2557
- SourceFileUtcTime: "",
2558
- SourceFileSize: -1,
2559
- StaleUtcTime: "",
2560
- TableName: "",
2561
- Fields: {
2562
- QvdFieldHeader: this._columns.map((column) => ({
2563
- FieldName: column,
2564
- BitOffset: 0,
2565
- BitWidth: 0,
2566
- Bias: 0,
2567
- NoOfSymbols: 0,
2568
- Offset: 0,
2569
- Length: 0,
2570
- Comment: "",
2571
- NumberFormat: {
2572
- Type: "UNKNOWN",
2573
- nDec: "0",
2574
- UseThou: "0",
2575
- Fmt: "",
2576
- Dec: "",
2577
- Thou: ""
2578
- },
2579
- Tags: {}
2580
- }))
2581
- },
2582
- NoOfRecords: 0,
2583
- RecordByteSize: 0,
2584
- Offset: 0,
2585
- Length: 0,
2586
- Compression: "",
2587
- Comment: "",
2588
- EncryptionInfo: "",
2589
- TableTags: "",
2590
- ProfilingData: "",
2591
- Lineage: {}
2592
- };
2593
- }
4026
+ const header = this._ownMetadata();
2594
4027
  const modifiableFields = [
2595
4028
  "qvBuildNo",
2596
4029
  "creatorDoc",
@@ -2625,7 +4058,7 @@ var init_QvdDataFrame = __esm({
2625
4058
  };
2626
4059
  modifiableFields.forEach((field) => {
2627
4060
  if (metadata[field] !== void 0) {
2628
- this._metadata[fieldMapping[field]] = metadata[field];
4061
+ header[fieldMapping[field]] = metadata[field];
2629
4062
  }
2630
4063
  });
2631
4064
  }
@@ -2638,30 +4071,44 @@ var init_QvdDataFrame = __esm({
2638
4071
  /**
2639
4072
  * Sets modifiable field-level metadata for a specific field.
2640
4073
  * Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
4074
+ *
4075
+ * Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
4076
+ * header does not describe. The change applies to this frame only, never to a frame it was derived
4077
+ * from or shares a header with.
4078
+ *
2641
4079
  * @param {string} fieldName The name of the field.
2642
4080
  * @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
4081
+ * @throws {QvdValidationError} If the frame has no column of that name.
2643
4082
  */
2644
4083
  setFieldMetadata(fieldName, metadata) {
2645
- if (!this._metadata || !this._metadata.Fields || !this._metadata.Fields.QvdFieldHeader) {
2646
- return;
4084
+ if (!this._columns.includes(fieldName)) {
4085
+ throw new exports.QvdValidationError(`Column '${fieldName}' does not exist`, {
4086
+ column: fieldName,
4087
+ availableColumns: this._columns
4088
+ });
2647
4089
  }
2648
- let fields = this._metadata.Fields.QvdFieldHeader;
4090
+ const header = this._ownMetadata();
4091
+ if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
4092
+ header.Fields = { QvdFieldHeader: [] };
4093
+ }
4094
+ let fields = header.Fields.QvdFieldHeader;
2649
4095
  if (!Array.isArray(fields)) {
2650
4096
  fields = [fields];
2651
- this._metadata.Fields.QvdFieldHeader = fields;
4097
+ header.Fields.QvdFieldHeader = fields;
2652
4098
  }
2653
- const fieldIndex = fields.findIndex((f) => f.FieldName === fieldName);
2654
- if (fieldIndex === -1) {
2655
- return;
4099
+ let field = fields.find((f) => f.FieldName === fieldName);
4100
+ if (!field) {
4101
+ field = defaultFieldHeader(fieldName);
4102
+ fields.push(field);
2656
4103
  }
2657
4104
  if (metadata.comment !== void 0) {
2658
- fields[fieldIndex].Comment = metadata.comment;
4105
+ field.Comment = metadata.comment;
2659
4106
  }
2660
4107
  if (metadata.numberFormat !== void 0) {
2661
- fields[fieldIndex].NumberFormat = metadata.numberFormat;
4108
+ field.NumberFormat = metadata.numberFormat;
2662
4109
  }
2663
4110
  if (metadata.tags !== void 0) {
2664
- fields[fieldIndex].Tags = metadata.tags;
4111
+ field.Tags = metadata.tags;
2665
4112
  }
2666
4113
  }
2667
4114
  /**
@@ -2678,7 +4125,7 @@ var init_QvdDataFrame = __esm({
2678
4125
  type: typeof n
2679
4126
  });
2680
4127
  }
2681
- return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
4128
+ return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
2682
4129
  }
2683
4130
  /**
2684
4131
  * Returns the last n rows of the data frame.
@@ -2694,7 +4141,13 @@ var init_QvdDataFrame = __esm({
2694
4141
  type: typeof n
2695
4142
  });
2696
4143
  }
2697
- return new _QvdDataFrame(n === 0 ? [] : this._data.slice(-n), this._columns, this._metadata);
4144
+ return new _QvdDataFrame(
4145
+ n === 0 ? [] : this._data.slice(-n),
4146
+ this._columns,
4147
+ this._metadata,
4148
+ null,
4149
+ this._storedSymbols
4150
+ );
2698
4151
  }
2699
4152
  /**
2700
4153
  * Returns the selected rows of the data frame.
@@ -2722,7 +4175,9 @@ var init_QvdDataFrame = __esm({
2722
4175
  return new _QvdDataFrame(
2723
4176
  args.map((index) => this._data[index]),
2724
4177
  this._columns,
2725
- this._metadata
4178
+ this._metadata,
4179
+ null,
4180
+ this._storedSymbols
2726
4181
  );
2727
4182
  }
2728
4183
  /**
@@ -2734,6 +4189,50 @@ var init_QvdDataFrame = __esm({
2734
4189
  * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
2735
4190
  */
2736
4191
  at(row, column) {
4192
+ const index = this._cellIndex(row, column);
4193
+ return this._data[row][index];
4194
+ }
4195
+ /**
4196
+ * Returns the text of the value at the specified row and column.
4197
+ *
4198
+ * The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
4199
+ * `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
4200
+ * or the string it was spelled as - and is null for a number that was stored as a pure number.
4201
+ *
4202
+ * ```js
4203
+ * const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
4204
+ * df.at(0, 'date'); // -52593
4205
+ * df.textAt(0, 'date'); // '1756-01-01'
4206
+ * ```
4207
+ *
4208
+ * @param {number} row The index of the row.
4209
+ * @param {string} column The name of the column.
4210
+ * @return {string|null} The text, or null for NULL and for a number with no text.
4211
+ * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
4212
+ */
4213
+ textAt(row, column) {
4214
+ const index = this._cellIndex(row, column);
4215
+ const value = this._data[row][index];
4216
+ if (typeof value === "string") {
4217
+ return value;
4218
+ }
4219
+ if (typeof value === "number") {
4220
+ const entry = storedSymbolsEntry(this._storedSymbols, column);
4221
+ return entry === null ? null : storedTextOf(entry, value);
4222
+ }
4223
+ const dual = asDual(value);
4224
+ return dual !== null && typeof dual.text === "string" ? dual.text : null;
4225
+ }
4226
+ /**
4227
+ * Checks a row and a column name, and returns the column's position.
4228
+ *
4229
+ * @param {number} row The index of the row.
4230
+ * @param {string} column The name of the column.
4231
+ * @return {number} The column's position.
4232
+ * @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
4233
+ * @private
4234
+ */
4235
+ _cellIndex(row, column) {
2737
4236
  if (typeof row !== "number" || !Number.isInteger(row)) {
2738
4237
  throw new exports.QvdValidationError("Row index must be an integer", {
2739
4238
  provided: row,
@@ -2753,7 +4252,7 @@ var init_QvdDataFrame = __esm({
2753
4252
  availableColumns: this._columns
2754
4253
  });
2755
4254
  }
2756
- return this._data[row][this._columns.indexOf(column)];
4255
+ return this._columns.indexOf(column);
2757
4256
  }
2758
4257
  /**
2759
4258
  * Selects the specified columns from the data frame.
@@ -2774,15 +4273,34 @@ var init_QvdDataFrame = __esm({
2774
4273
  const indices = args.map((arg) => this._columns.indexOf(arg));
2775
4274
  const data = this._data.map((row) => indices.map((index) => row[index]));
2776
4275
  const columns = indices.map((index) => this._columns[index]);
2777
- return new _QvdDataFrame(data, columns, this._metadata);
4276
+ return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
2778
4277
  }
2779
4278
  /**
2780
4279
  * Returns the data frame as a dictionary.
2781
4280
  *
2782
- * @return {Promise<{columns: Array<string>, data: Array<Array<any>>}>} The data frame as a dictionary.
4281
+ * Everything a frame needs to write the same file again, as plain data: the header, and the
4282
+ * stored-symbol record that says what cells showing one half of a symbol were read from. So
4283
+ * `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
4284
+ * `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
4285
+ *
4286
+ * @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
2783
4287
  */
2784
4288
  async toDict() {
2785
- return { columns: this._columns, data: this._data };
4289
+ return this.toJSON();
4290
+ }
4291
+ /**
4292
+ * The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
4293
+ * `fromDict(JSON.parse(...))` can revive.
4294
+ *
4295
+ * @return {QvdDataFrameDict} The data frame as a dictionary.
4296
+ */
4297
+ toJSON() {
4298
+ return {
4299
+ columns: this._columns,
4300
+ data: this._data,
4301
+ metadata: this._metadata,
4302
+ storedSymbols: this._storedSymbols
4303
+ };
2786
4304
  }
2787
4305
  /**
2788
4306
  * Persists the data frame to a QVD file.
@@ -2811,6 +4329,33 @@ var init_QvdDataFrame = __esm({
2811
4329
  * @param {Object} [options] Optional loading options.
2812
4330
  * @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
2813
4331
  * integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
4332
+ * This is the older name for `limit`; the two are the same option and passing both throws.
4333
+ * @param {number|null} [options.limit] Rows to read, counting from `offset`. The same number as
4334
+ * `maxRows`, spelled so that it reads correctly beside an offset.
4335
+ * @param {number} [options.offset=0] File row to start at. An offset past the end of the file
4336
+ * returns no rows rather than throwing, so a paging loop terminates on its own.
4337
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should
4338
+ * appear in the result. Unselected fields have their symbols skipped entirely rather than
4339
+ * parsed and discarded. An unknown or repeated name throws.
4340
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
4341
+ * text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
4342
+ * `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
4343
+ * `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
4344
+ * implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
4345
+ * `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
4346
+ * double, a string and NULL read the same in every mode: a number, a string and null. Anything else
4347
+ * throws.
4348
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
4349
+ * reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
4350
+ * reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
4351
+ * `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
4352
+ * `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
4353
+ * original string or dual back, and a value that two stored values read as is refused there
4354
+ * rather than written as either. Anything but a boolean throws.
4355
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
4356
+ * read proceeds - the same shape `toQvd`'s callback receives.
4357
+ * @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
4358
+ * which is a `DOMException` named `AbortError` unless you aborted with a reason of your own.
2814
4359
  * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
2815
4360
  * must be within this directory, with symlinks resolved first, so a link inside it that points
2816
4361
  * outside it is rejected. Defaults to the current working directory. To permit an entire
@@ -2823,17 +4368,64 @@ var init_QvdDataFrame = __esm({
2823
4368
  * **Zero disables the memory check entirely.**
2824
4369
  * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
2825
4370
  * a lazy load switches to the two-pass filtering path. Defaults to 50MB.
2826
- * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
4371
+ * @throws {QvdValidationError} If a window option is not a non-negative integer, if both
4372
+ * `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
4373
+ * is not one of its modes, or if `coerceNumericStrings` is not a boolean.
2827
4374
  * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
2828
4375
  */
2829
4376
  static async fromQvd(path3, options = {}) {
2830
4377
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
2831
- const readerOptions = {
2832
- allowedDir: options.allowedDir,
2833
- memorySafetyFactor: options.memorySafetyFactor,
2834
- symbolFilteringThreshold: options.symbolFilteringThreshold
2835
- };
2836
- return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
4378
+ return await new QvdFileReader2(path3, readerOptionsFrom(options)).load(windowFrom(options));
4379
+ }
4380
+ /**
4381
+ * Reads a QVD file in chunks, as an async generator of data frames.
4382
+ *
4383
+ * The file is opened, read and parsed once; only the index decode and the row building happen
4384
+ * per chunk, so what this bounds is row materialisation - the part that actually dominates a
4385
+ * large read's heap. It is **not** constant-memory reading of an arbitrarily large file: the
4386
+ * symbol table is parsed in full whatever the chunk size, because a stored index in the last
4387
+ * chunk can address the first symbol. On a high-cardinality file that table is the bulk of the
4388
+ * cost, and `readMetadata` is the only read that avoids it.
4389
+ *
4390
+ * ```js
4391
+ * for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
4392
+ * process(chunk.data);
4393
+ * }
4394
+ * ```
4395
+ *
4396
+ * A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
4397
+ * run its body.
4398
+ *
4399
+ * Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
4400
+ * them, so a chunk written on its own writes the symbols its cells were read from.
4401
+ *
4402
+ * @param {string} path The path to the QVD file.
4403
+ * @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
4404
+ * @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
4405
+ * @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
4406
+ * @param {number|null} [options.limit] Rows to cover, counting from `offset`.
4407
+ * @param {number} [options.offset=0] File row to start at.
4408
+ * @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
4409
+ * @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
4410
+ * its text, or a frozen `QvdDual` holding both. Anything else throws.
4411
+ * @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
4412
+ * reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
4413
+ * `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
4414
+ * `storedSymbols`. Anything but a boolean throws.
4415
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
4416
+ * over the rows counts the whole window, not each chunk.
4417
+ * @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
4418
+ * @param {string} [options.allowedDir] Directory the path must resolve inside.
4419
+ * @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
4420
+ * charged for two chunks of rows rather than the window. Zero disables the check.
4421
+ * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
4422
+ * windowed read switches to two-pass filtering.
4423
+ * @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
4424
+ */
4425
+ static async *iterate(path3, options = {}) {
4426
+ const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
4427
+ const reader = new QvdFileReader2(path3, readerOptionsFrom(options));
4428
+ yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
2837
4429
  }
2838
4430
  /**
2839
4431
  * Reads a QVD file's schema and header metadata, without reading its data.
@@ -2857,17 +4449,28 @@ var init_QvdDataFrame = __esm({
2857
4449
  * @param {Object} [options] Optional reading options.
2858
4450
  * @param {string} [options.allowedDir] Optional allowed directory path, applied exactly as it
2859
4451
  * is for `fromQvd`.
4452
+ * @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`, as on
4453
+ * the reads that return data. Only the `read` and `header` stages occur here; there are no
4454
+ * symbols to parse and no rows to build.
4455
+ * @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
2860
4456
  * @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
2861
4457
  */
2862
4458
  static async readMetadata(path3, options = {}) {
2863
4459
  const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
2864
- return await new QvdFileReader2(path3, { allowedDir: options.allowedDir }).loadMetadata();
4460
+ return await new QvdFileReader2(path3, metadataOptionsFrom(options)).loadMetadata();
2865
4461
  }
2866
4462
  /**
2867
4463
  * Constructs a data frame from a dictionary.
2868
4464
  *
2869
- * @param {{columns: Array<string>, data: Array<Array<any>>}} data The dictionary to construct the data frame from.
4465
+ * Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
4466
+ * rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
4467
+ * spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
4468
+ * here rather than written.
4469
+ *
4470
+ * @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
2870
4471
  * @return {Promise<QvdDataFrame>} The constructed data frame.
4472
+ * @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
4473
+ * or `storedSymbols` is malformed or names a field that is not one of the columns.
2871
4474
  */
2872
4475
  static async fromDict(data) {
2873
4476
  if (!data.columns) {
@@ -2880,18 +4483,292 @@ var init_QvdDataFrame = __esm({
2880
4483
  data
2881
4484
  });
2882
4485
  }
2883
- return new _QvdDataFrame(data.data, data.columns);
4486
+ const { metadata = null, storedSymbols = null } = data;
4487
+ if (metadata !== null && !isPlainObject(metadata)) {
4488
+ throw new exports.QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
4489
+ type: typeof metadata
4490
+ });
4491
+ }
4492
+ const record = normaliseStoredSymbols(storedSymbols);
4493
+ if (record !== null) {
4494
+ const unknown = record.find((entry) => !data.columns.includes(entry.field));
4495
+ if (unknown !== void 0) {
4496
+ throw new exports.QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
4497
+ field: unknown.field,
4498
+ availableColumns: data.columns
4499
+ });
4500
+ }
4501
+ }
4502
+ return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
2884
4503
  }
2885
4504
  };
2886
4505
  }
2887
4506
  });
2888
4507
 
4508
+ // src/QvdSymbol.js
4509
+ init_QvdErrors();
4510
+ init_cellRules();
4511
+ init_symbolBytes();
4512
+ function checkInteger(value, context) {
4513
+ if (typeof value === "number" && isStoredAsInt(value)) {
4514
+ return;
4515
+ }
4516
+ throw new exports.QvdValidationError(
4517
+ `The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
4518
+ { ...context, half: "integer", type: typeof value }
4519
+ );
4520
+ }
4521
+ __name(checkInteger, "checkInteger");
4522
+ var QvdSymbol = class _QvdSymbol {
4523
+ static {
4524
+ __name(this, "QvdSymbol");
4525
+ }
4526
+ /**
4527
+ * Constructs a new QVD symbol.
4528
+ *
4529
+ * @param {number|null} intValue The integer value.
4530
+ * @param {number|null} doubleValue The double value.
4531
+ * @param {string|null} stringValue The string value.
4532
+ */
4533
+ constructor(intValue, doubleValue, stringValue) {
4534
+ this._intValue = intValue;
4535
+ this._doubleValue = doubleValue;
4536
+ this._stringValue = stringValue;
4537
+ }
4538
+ /**
4539
+ * Returns the integer value of this symbol.
4540
+ *
4541
+ * @return {number|null} The integer value.
4542
+ */
4543
+ get intValue() {
4544
+ return this._intValue;
4545
+ }
4546
+ /**
4547
+ * Returns the double value of this symbol.
4548
+ *
4549
+ * @return {number|null} The double value.
4550
+ */
4551
+ get doubleValue() {
4552
+ return this._doubleValue;
4553
+ }
4554
+ /**
4555
+ * Returns the string value of this symbol.
4556
+ *
4557
+ * @return {string|null} The string value.
4558
+ */
4559
+ get stringValue() {
4560
+ return this._stringValue;
4561
+ }
4562
+ /**
4563
+ * Retrieves the primary value of this symbol. The primary value is descriptive raw value.
4564
+ * It is either the string value, the integer value or the double value, prioritized in this order.
4565
+ *
4566
+ * @return {number|string|null} The primary value.
4567
+ */
4568
+ toPrimaryValue() {
4569
+ if (null != this._stringValue) {
4570
+ return this._stringValue;
4571
+ } else if (null != this._intValue) {
4572
+ return this._intValue;
4573
+ } else if (null != this._doubleValue) {
4574
+ return this._doubleValue;
4575
+ } else {
4576
+ return null;
4577
+ }
4578
+ }
4579
+ /**
4580
+ * Converts the symbol to its byte representation.
4581
+ *
4582
+ * The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
4583
+ * double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
4584
+ * checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
4585
+ * outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
4586
+ * integer and a double silently lost the double, a text with a NUL produced a symbol that ends
4587
+ * early, and a text with an unpaired surrogate was written with U+FFFD in its place.
4588
+ *
4589
+ * A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
4590
+ * `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
4591
+ * `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
4592
+ *
4593
+ * @return {Buffer} The byte representation of the symbol.
4594
+ * @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
4595
+ * holds a half no symbol can store. The message names the half.
4596
+ */
4597
+ toByteRepresentation() {
4598
+ const intValue = this._intValue ?? null;
4599
+ const doubleValue = this._doubleValue ?? null;
4600
+ const stringValue = this._stringValue ?? null;
4601
+ if (intValue !== null && doubleValue !== null) {
4602
+ throw new exports.QvdValidationError("A symbol holds an integer or a double, not both", {
4603
+ intValue,
4604
+ doubleValue
4605
+ });
4606
+ }
4607
+ if (intValue === null && doubleValue === null && stringValue === null) {
4608
+ throw new exports.QvdValidationError("The symbol does not contain any value.", {
4609
+ intValue,
4610
+ doubleValue,
4611
+ stringValue
4612
+ });
4613
+ }
4614
+ if (intValue !== null) {
4615
+ checkInteger(intValue, {});
4616
+ }
4617
+ if (doubleValue !== null) {
4618
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4619
+ }
4620
+ if (stringValue !== null) {
4621
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4622
+ }
4623
+ const number = intValue ?? doubleValue;
4624
+ const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
4625
+ const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
4626
+ writeSymbol(buffer, 0, kind, number, stringValue);
4627
+ return buffer;
4628
+ }
4629
+ /**
4630
+ * Checks if this symbol is equal to another symbol.
4631
+ *
4632
+ * By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
4633
+ * `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
4634
+ * answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
4635
+ * or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
4636
+ * storage kind, so either kind matches.
4637
+ *
4638
+ * @param {*} value The object to compare with.
4639
+ * @return {boolean} True if the objects are equal, false otherwise.
4640
+ */
4641
+ equals(value) {
4642
+ if (value === null || typeof value !== "object") {
4643
+ return false;
4644
+ }
4645
+ const intValue = this._intValue ?? null;
4646
+ const doubleValue = this._doubleValue ?? null;
4647
+ const stringValue = this._stringValue ?? null;
4648
+ const dual = asDual(value);
4649
+ if (dual !== null) {
4650
+ return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
4651
+ }
4652
+ if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
4653
+ return false;
4654
+ }
4655
+ return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
4656
+ }
4657
+ /**
4658
+ * Constructs a pure integer value symbol.
4659
+ *
4660
+ * @param {number} intValue The integer value.
4661
+ * @return {QvdSymbol} The constructed value symbol.
4662
+ * @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
4663
+ */
4664
+ static fromIntValue(intValue) {
4665
+ checkInteger(intValue, {});
4666
+ return new _QvdSymbol(intValue, null, null);
4667
+ }
4668
+ /**
4669
+ * Constructs a pure double value symbol.
4670
+ *
4671
+ * @param {number} doubleValue The double value.
4672
+ * @return {QvdSymbol} The constructed value symbol.
4673
+ * @throws {QvdValidationError} If the double is not a finite number.
4674
+ */
4675
+ static fromDoubleValue(doubleValue) {
4676
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4677
+ return new _QvdSymbol(null, doubleValue, null);
4678
+ }
4679
+ /**
4680
+ * Constructs a pure string value symbol.
4681
+ *
4682
+ * @param {string} stringValue The string value.
4683
+ * @return {QvdSymbol} The constructed value symbol.
4684
+ * @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
4685
+ */
4686
+ static fromStringValue(stringValue) {
4687
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4688
+ return new _QvdSymbol(null, null, stringValue);
4689
+ }
4690
+ /**
4691
+ * Constructs a dual value symbol from an integer and a string value.
4692
+ *
4693
+ * @param {number} intValue The integer value.
4694
+ * @param {string} stringValue The string value.
4695
+ * @return {QvdSymbol} The constructed value symbol.
4696
+ * @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
4697
+ * is not a string, or holds a NUL or an unpaired surrogate.
4698
+ */
4699
+ static fromDualIntValue(intValue, stringValue) {
4700
+ checkInteger(intValue, {});
4701
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4702
+ return new _QvdSymbol(intValue, null, stringValue);
4703
+ }
4704
+ /**
4705
+ * Constructs a dual value symbol from a double and a string value.
4706
+ *
4707
+ * @param {number} doubleValue The double value.
4708
+ * @param {string} stringValue The string value.
4709
+ * @return {QvdSymbol} The constructed value symbol.
4710
+ * @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
4711
+ * or holds a NUL or an unpaired surrogate.
4712
+ */
4713
+ static fromDualDoubleValue(doubleValue, stringValue) {
4714
+ checkNumber(doubleValue, "The double of a symbol", { half: "double" });
4715
+ checkText(stringValue, "The text of a symbol", { half: "text" });
4716
+ return new _QvdSymbol(null, doubleValue, stringValue);
4717
+ }
4718
+ };
4719
+
4720
+ // src/index.js
4721
+ init_QvdDual();
4722
+
4723
+ // src/util/qlikDate.js
4724
+ init_QvdErrors();
4725
+ init_cellRules();
4726
+ var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
4727
+ var MS_PER_DAY = 864e5;
4728
+ var MAX_DATE_MS = 864e13;
4729
+ var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
4730
+ var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
4731
+ function qlikSerialToDate(serial) {
4732
+ const dual = asDual(serial);
4733
+ const number = dual === null ? serial : dual.number;
4734
+ if (typeof number !== "number" || !Number.isFinite(number)) {
4735
+ throw new exports.QvdValidationError("A Qlik date serial must be a finite number", {
4736
+ provided: describeType(number),
4737
+ type: typeof serial
4738
+ });
4739
+ }
4740
+ const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
4741
+ if (!(Math.abs(ms) <= MAX_DATE_MS)) {
4742
+ throw new exports.QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
4743
+ serial: number,
4744
+ minSerial: MIN_SERIAL,
4745
+ maxSerial: MAX_SERIAL
4746
+ });
4747
+ }
4748
+ return new Date(ms);
4749
+ }
4750
+ __name(qlikSerialToDate, "qlikSerialToDate");
4751
+ function dateToQlikSerial(date) {
4752
+ const ms = util.types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
4753
+ if (Number.isNaN(ms)) {
4754
+ throw new exports.QvdValidationError("dateToQlikSerial needs a valid Date", {
4755
+ provided: describeType(date),
4756
+ type: typeof date
4757
+ });
4758
+ }
4759
+ return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
4760
+ }
4761
+ __name(dateToQlikSerial, "dateToQlikSerial");
4762
+
2889
4763
  // src/index.js
2890
- init_QvdSymbol();
2891
4764
  init_QvdDataFrame();
2892
4765
  init_QvdColumnTable();
2893
4766
  init_QvdFileReader();
2894
4767
  init_QvdFileWriter();
2895
4768
  init_QvdErrors();
4769
+
4770
+ exports.QvdSymbol = QvdSymbol;
4771
+ exports.dateToQlikSerial = dateToQlikSerial;
4772
+ exports.qlikSerialToDate = qlikSerialToDate;
2896
4773
  //# sourceMappingURL=index.cjs.map
2897
4774
  //# sourceMappingURL=index.cjs.map