qvdjs 0.10.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -1101
- package/dist/index.cjs +2612 -735
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2610 -737
- package/dist/index.js.map +1 -1
- package/img/logo/qvdjs_logo-512.png +0 -0
- package/package.json +6 -2
package/dist/index.cjs
CHANGED
|
@@ -7,6 +7,7 @@ var xml2 = require('xml2js');
|
|
|
7
7
|
var assert2 = require('assert');
|
|
8
8
|
var os = require('os');
|
|
9
9
|
var v8 = require('v8');
|
|
10
|
+
var util = require('util');
|
|
10
11
|
|
|
11
12
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
12
13
|
|
|
@@ -20,6 +21,7 @@ var v8__default = /*#__PURE__*/_interopDefault(v8);
|
|
|
20
21
|
|
|
21
22
|
var __defProp = Object.defineProperty;
|
|
22
23
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
24
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
23
25
|
var __esm = (fn, res) => function __init() {
|
|
24
26
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
25
27
|
};
|
|
@@ -29,10 +31,14 @@ var __export = (target, all) => {
|
|
|
29
31
|
};
|
|
30
32
|
|
|
31
33
|
// src/QvdErrors.js
|
|
32
|
-
exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
|
|
34
|
+
var ERROR_NAMES; exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
|
|
33
35
|
var init_QvdErrors = __esm({
|
|
34
36
|
"src/QvdErrors.js"() {
|
|
37
|
+
ERROR_NAMES = /* @__PURE__ */ new Map();
|
|
35
38
|
exports.QvdError = class extends Error {
|
|
39
|
+
static {
|
|
40
|
+
__name(this, "QvdError");
|
|
41
|
+
}
|
|
36
42
|
/**
|
|
37
43
|
* Constructs a new QVD error.
|
|
38
44
|
*
|
|
@@ -42,13 +48,16 @@ var init_QvdErrors = __esm({
|
|
|
42
48
|
*/
|
|
43
49
|
constructor(message, code, context = {}) {
|
|
44
50
|
super(message);
|
|
45
|
-
this.name =
|
|
51
|
+
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
46
52
|
this.code = code;
|
|
47
53
|
this.context = context;
|
|
48
54
|
Error.captureStackTrace(this, this.constructor);
|
|
49
55
|
}
|
|
50
56
|
};
|
|
51
57
|
exports.QvdParseError = class extends exports.QvdError {
|
|
58
|
+
static {
|
|
59
|
+
__name(this, "QvdParseError");
|
|
60
|
+
}
|
|
52
61
|
/**
|
|
53
62
|
* Constructs a new QVD parse error.
|
|
54
63
|
*
|
|
@@ -60,6 +69,9 @@ var init_QvdErrors = __esm({
|
|
|
60
69
|
}
|
|
61
70
|
};
|
|
62
71
|
exports.QvdValidationError = class extends exports.QvdError {
|
|
72
|
+
static {
|
|
73
|
+
__name(this, "QvdValidationError");
|
|
74
|
+
}
|
|
63
75
|
/**
|
|
64
76
|
* Constructs a new QVD validation error.
|
|
65
77
|
*
|
|
@@ -71,6 +83,9 @@ var init_QvdErrors = __esm({
|
|
|
71
83
|
}
|
|
72
84
|
};
|
|
73
85
|
exports.QvdIOError = class extends exports.QvdError {
|
|
86
|
+
static {
|
|
87
|
+
__name(this, "QvdIOError");
|
|
88
|
+
}
|
|
74
89
|
/**
|
|
75
90
|
* Constructs a new QVD IO error.
|
|
76
91
|
*
|
|
@@ -82,6 +97,9 @@ var init_QvdErrors = __esm({
|
|
|
82
97
|
}
|
|
83
98
|
};
|
|
84
99
|
exports.QvdCorruptedError = class extends exports.QvdError {
|
|
100
|
+
static {
|
|
101
|
+
__name(this, "QvdCorruptedError");
|
|
102
|
+
}
|
|
85
103
|
/**
|
|
86
104
|
* Constructs a new QVD corrupted error.
|
|
87
105
|
*
|
|
@@ -93,6 +111,9 @@ var init_QvdErrors = __esm({
|
|
|
93
111
|
}
|
|
94
112
|
};
|
|
95
113
|
exports.QvdSecurityError = class extends exports.QvdError {
|
|
114
|
+
static {
|
|
115
|
+
__name(this, "QvdSecurityError");
|
|
116
|
+
}
|
|
96
117
|
/**
|
|
97
118
|
* Constructs a new QVD security error.
|
|
98
119
|
*
|
|
@@ -103,163 +124,591 @@ var init_QvdErrors = __esm({
|
|
|
103
124
|
super(message, "QVD_SECURITY_ERROR", context);
|
|
104
125
|
}
|
|
105
126
|
};
|
|
127
|
+
ERROR_NAMES.set(exports.QvdError, "QvdError");
|
|
128
|
+
ERROR_NAMES.set(exports.QvdParseError, "QvdParseError");
|
|
129
|
+
ERROR_NAMES.set(exports.QvdValidationError, "QvdValidationError");
|
|
130
|
+
ERROR_NAMES.set(exports.QvdIOError, "QvdIOError");
|
|
131
|
+
ERROR_NAMES.set(exports.QvdCorruptedError, "QvdCorruptedError");
|
|
132
|
+
ERROR_NAMES.set(exports.QvdSecurityError, "QvdSecurityError");
|
|
106
133
|
}
|
|
107
134
|
});
|
|
108
135
|
|
|
109
|
-
// src/
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
136
|
+
// src/util/cellRules.js
|
|
137
|
+
function asDual(value) {
|
|
138
|
+
if (value === null || typeof value !== "object") {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
try {
|
|
142
|
+
if (value[DUAL_BRAND] === true) {
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
if (!isPlainObject(value)) {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
const keys = Object.keys(value);
|
|
149
|
+
return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
|
|
150
|
+
} catch {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function isPlainObject(value) {
|
|
155
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
const prototype = Object.getPrototypeOf(value);
|
|
159
|
+
return prototype === null || Object.getPrototypeOf(prototype) === null;
|
|
160
|
+
}
|
|
161
|
+
function isNumericText(text) {
|
|
162
|
+
return text.trim() !== "" && Number.isFinite(Number(text));
|
|
163
|
+
}
|
|
164
|
+
function isStoredAsInt(value) {
|
|
165
|
+
return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
|
|
166
|
+
}
|
|
167
|
+
function numberProblem(value) {
|
|
168
|
+
if (typeof value !== "number") {
|
|
169
|
+
return `is ${describeType(value)}, not a number`;
|
|
170
|
+
}
|
|
171
|
+
return Number.isFinite(value) ? null : `is ${String(value)}`;
|
|
172
|
+
}
|
|
173
|
+
function textProblem(value) {
|
|
174
|
+
if (typeof value !== "string") {
|
|
175
|
+
return { reason: "type" };
|
|
176
|
+
}
|
|
177
|
+
const position = value.indexOf(NUL);
|
|
178
|
+
if (position !== -1) {
|
|
179
|
+
return { reason: "nul", position };
|
|
180
|
+
}
|
|
181
|
+
return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
|
|
182
|
+
}
|
|
183
|
+
function unpairedSurrogateIndex(value) {
|
|
184
|
+
for (let index = 0; index < value.length; index++) {
|
|
185
|
+
const unit = value.charCodeAt(index);
|
|
186
|
+
if (unit < 55296 || unit > 57343) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const next = value.charCodeAt(index + 1);
|
|
190
|
+
if (unit <= 56319 && next >= 56320 && next <= 57343) {
|
|
191
|
+
index++;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
return index;
|
|
195
|
+
}
|
|
196
|
+
return -1;
|
|
197
|
+
}
|
|
198
|
+
function checkNumber(value, subject, context) {
|
|
199
|
+
if (numberProblem(value) === null) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (subject === null && typeof value === "number") {
|
|
203
|
+
throw new exports.QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
|
|
204
|
+
...context,
|
|
205
|
+
provided: String(value)
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
throw new exports.QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
|
|
209
|
+
...context,
|
|
210
|
+
type: typeof value
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
function checkText(value, subject, context) {
|
|
214
|
+
const problem = textProblem(value);
|
|
215
|
+
if (problem === null) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (problem.reason === "type") {
|
|
219
|
+
throw new exports.QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
|
|
220
|
+
...context,
|
|
221
|
+
type: typeof value
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
if (problem.reason === "surrogate") {
|
|
225
|
+
throw new exports.QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
|
|
226
|
+
...context,
|
|
227
|
+
position: problem.position
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
throw new exports.QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
|
|
231
|
+
}
|
|
232
|
+
function constructorName(value) {
|
|
233
|
+
try {
|
|
234
|
+
const name = value.constructor?.name;
|
|
235
|
+
return typeof name === "string" && name !== "" ? name : null;
|
|
236
|
+
} catch {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
function describeType(value) {
|
|
241
|
+
if (value === null) {
|
|
242
|
+
return "null";
|
|
243
|
+
}
|
|
244
|
+
if (typeof value === "number") {
|
|
245
|
+
return Number.isFinite(value) ? "a number" : String(value);
|
|
246
|
+
}
|
|
247
|
+
if (typeof value === "undefined") {
|
|
248
|
+
return "undefined";
|
|
249
|
+
}
|
|
250
|
+
if (typeof value !== "object") {
|
|
251
|
+
return `a ${typeof value}`;
|
|
252
|
+
}
|
|
253
|
+
let name;
|
|
254
|
+
try {
|
|
255
|
+
if (Array.isArray(value)) {
|
|
256
|
+
return "an array";
|
|
257
|
+
}
|
|
258
|
+
name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
|
|
259
|
+
if (name === "Object") {
|
|
260
|
+
if (!isPlainObject(value)) {
|
|
261
|
+
return "an object whose prototype is not Object.prototype";
|
|
262
|
+
}
|
|
263
|
+
const keys = Object.keys(value);
|
|
264
|
+
if (keys.length === 0) {
|
|
265
|
+
return "a plain object";
|
|
266
|
+
}
|
|
267
|
+
return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
|
|
268
|
+
}
|
|
269
|
+
} catch {
|
|
270
|
+
return "an object";
|
|
271
|
+
}
|
|
272
|
+
return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
|
|
273
|
+
}
|
|
274
|
+
var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
|
|
275
|
+
var init_cellRules = __esm({
|
|
276
|
+
"src/util/cellRules.js"() {
|
|
113
277
|
init_QvdErrors();
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
278
|
+
INT32_MIN = -2147483648;
|
|
279
|
+
INT32_MAX = 2147483647;
|
|
280
|
+
NUL = String.fromCharCode(0);
|
|
281
|
+
DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
|
|
282
|
+
__name(asDual, "asDual");
|
|
283
|
+
__name(isPlainObject, "isPlainObject");
|
|
284
|
+
__name(isNumericText, "isNumericText");
|
|
285
|
+
__name(isStoredAsInt, "isStoredAsInt");
|
|
286
|
+
__name(numberProblem, "numberProblem");
|
|
287
|
+
__name(textProblem, "textProblem");
|
|
288
|
+
__name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
|
|
289
|
+
__name(checkNumber, "checkNumber");
|
|
290
|
+
__name(checkText, "checkText");
|
|
291
|
+
__name(constructorName, "constructorName");
|
|
292
|
+
__name(describeType, "describeType");
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
// src/util/symbolBytes.js
|
|
297
|
+
function kindOf(number, text) {
|
|
298
|
+
if (number === null) {
|
|
299
|
+
return 4;
|
|
300
|
+
}
|
|
301
|
+
if (isStoredAsInt(number)) {
|
|
302
|
+
return text === null ? 1 : 5;
|
|
303
|
+
}
|
|
304
|
+
return text === null ? 2 : 6;
|
|
305
|
+
}
|
|
306
|
+
function symbolByteLength(kind, number, text) {
|
|
307
|
+
const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
|
|
308
|
+
const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
|
|
309
|
+
return 1 + numberBytes + textBytes;
|
|
310
|
+
}
|
|
311
|
+
function writeSymbol(buffer, offset, kind, number, text) {
|
|
312
|
+
buffer[offset++] = kind;
|
|
313
|
+
if (kind === 1 || kind === 5) {
|
|
314
|
+
offset = buffer.writeInt32LE(number, offset);
|
|
315
|
+
} else if (kind === 2 || kind === 6) {
|
|
316
|
+
offset = buffer.writeDoubleLE(number, offset);
|
|
317
|
+
}
|
|
318
|
+
if (kind >= 4) {
|
|
319
|
+
offset += buffer.write(text, offset, "utf8");
|
|
320
|
+
buffer[offset++] = 0;
|
|
321
|
+
}
|
|
322
|
+
return offset;
|
|
323
|
+
}
|
|
324
|
+
var init_symbolBytes = __esm({
|
|
325
|
+
"src/util/symbolBytes.js"() {
|
|
326
|
+
init_cellRules();
|
|
327
|
+
__name(kindOf, "kindOf");
|
|
328
|
+
__name(symbolByteLength, "symbolByteLength");
|
|
329
|
+
__name(writeSymbol, "writeSymbol");
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
// src/QvdDual.js
|
|
334
|
+
function defineHalves(target, number, text) {
|
|
335
|
+
Object.defineProperties(target, {
|
|
336
|
+
number: { value: number, enumerable: true },
|
|
337
|
+
text: { value: text, enumerable: true }
|
|
338
|
+
});
|
|
339
|
+
Object.freeze(target);
|
|
340
|
+
}
|
|
341
|
+
function dualFromSymbol(number, text) {
|
|
342
|
+
const dual = Object.create(exports.QvdDual.prototype);
|
|
343
|
+
defineHalves(dual, number, text);
|
|
344
|
+
return dual;
|
|
345
|
+
}
|
|
346
|
+
exports.QvdDual = void 0;
|
|
347
|
+
var init_QvdDual = __esm({
|
|
348
|
+
"src/QvdDual.js"() {
|
|
349
|
+
init_cellRules();
|
|
350
|
+
exports.QvdDual = class {
|
|
351
|
+
static {
|
|
352
|
+
__name(this, "QvdDual");
|
|
126
353
|
}
|
|
127
354
|
/**
|
|
128
|
-
*
|
|
355
|
+
* Constructs a dual value.
|
|
356
|
+
*
|
|
357
|
+
* The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
|
|
358
|
+
* integer inside the int32 range is stored as a dual int, anything else as a dual double.
|
|
129
359
|
*
|
|
130
|
-
* @
|
|
360
|
+
* @param {number} number The numeric half. Must be a finite number.
|
|
361
|
+
* @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
|
|
362
|
+
* @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
|
|
131
363
|
*/
|
|
132
|
-
|
|
133
|
-
|
|
364
|
+
constructor(number, text) {
|
|
365
|
+
checkNumber(number, "The number of a dual value", { half: "number" });
|
|
366
|
+
checkText(text, "The text of a dual value", { half: "text" });
|
|
367
|
+
defineHalves(this, number, text);
|
|
134
368
|
}
|
|
135
369
|
/**
|
|
136
|
-
*
|
|
370
|
+
* Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
|
|
137
371
|
*
|
|
138
|
-
*
|
|
372
|
+
* The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
|
|
373
|
+
* conversion ran, not which half the caller meant, and the fix is the same for all three.
|
|
374
|
+
*
|
|
375
|
+
* @return {never}
|
|
376
|
+
* @throws {TypeError} Always.
|
|
139
377
|
*/
|
|
140
|
-
|
|
141
|
-
|
|
378
|
+
[Symbol.toPrimitive]() {
|
|
379
|
+
throw new TypeError(
|
|
380
|
+
`A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
|
|
381
|
+
);
|
|
142
382
|
}
|
|
143
383
|
/**
|
|
144
|
-
*
|
|
384
|
+
* Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
|
|
145
385
|
*
|
|
146
|
-
* @return {string
|
|
386
|
+
* @return {{number: number, text: string}} The dual as a plain object.
|
|
147
387
|
*/
|
|
148
|
-
|
|
149
|
-
return this.
|
|
388
|
+
toJSON() {
|
|
389
|
+
return { number: this.number, text: this.text };
|
|
150
390
|
}
|
|
151
391
|
/**
|
|
152
|
-
*
|
|
153
|
-
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
392
|
+
* How Node's `util.inspect` and `console.log` show a dual.
|
|
154
393
|
*
|
|
155
|
-
* @return {
|
|
394
|
+
* @return {string} For example `QvdDual(4.5, "4.50")`.
|
|
156
395
|
*/
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
return this._stringValue;
|
|
160
|
-
} else if (null != this._intValue) {
|
|
161
|
-
return this._intValue;
|
|
162
|
-
} else if (null != this._doubleValue) {
|
|
163
|
-
return this._doubleValue;
|
|
164
|
-
} else {
|
|
165
|
-
return null;
|
|
166
|
-
}
|
|
396
|
+
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
397
|
+
return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
|
|
167
398
|
}
|
|
168
|
-
/**
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
* @return {Buffer} The byte representation of the symbol.
|
|
172
|
-
*/
|
|
173
|
-
toByteRepresentation() {
|
|
174
|
-
if (this._intValue !== null && this._stringValue !== null) {
|
|
175
|
-
const intBuffer = Buffer.alloc(4);
|
|
176
|
-
intBuffer.writeInt32LE(this._intValue);
|
|
177
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
178
|
-
return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
|
|
179
|
-
} else if (this._doubleValue !== null && this._stringValue !== null) {
|
|
180
|
-
const floatBuffer = Buffer.alloc(8);
|
|
181
|
-
floatBuffer.writeDoubleLE(this._doubleValue);
|
|
182
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
183
|
-
return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
|
|
184
|
-
} else if (this._intValue !== null) {
|
|
185
|
-
const buffer = Buffer.alloc(4);
|
|
186
|
-
buffer.writeInt32LE(this._intValue);
|
|
187
|
-
return Buffer.concat([Buffer.from([1]), buffer]);
|
|
188
|
-
} else if (this._doubleValue !== null) {
|
|
189
|
-
const buffer = Buffer.alloc(8);
|
|
190
|
-
buffer.writeDoubleLE(this._doubleValue);
|
|
191
|
-
return Buffer.concat([Buffer.from([2]), buffer]);
|
|
192
|
-
} else if (this._stringValue !== null) {
|
|
193
|
-
const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
194
|
-
return Buffer.concat([Buffer.from([4]), buffer]);
|
|
195
|
-
} else {
|
|
196
|
-
throw new exports.QvdValidationError("The symbol does not contain any value.", {
|
|
197
|
-
intValue: this._intValue,
|
|
198
|
-
doubleValue: this._doubleValue,
|
|
199
|
-
stringValue: this._stringValue
|
|
200
|
-
});
|
|
201
|
-
}
|
|
399
|
+
/** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
|
|
400
|
+
get [Symbol.toStringTag]() {
|
|
401
|
+
return "QvdDual";
|
|
202
402
|
}
|
|
203
403
|
/**
|
|
204
|
-
*
|
|
404
|
+
* Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
|
|
405
|
+
* own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
|
|
205
406
|
*
|
|
206
|
-
*
|
|
207
|
-
* @return {boolean} True if the objects are equal, false otherwise.
|
|
208
|
-
*/
|
|
209
|
-
equals(value) {
|
|
210
|
-
if (!(value instanceof _QvdSymbol)) {
|
|
211
|
-
return false;
|
|
212
|
-
}
|
|
213
|
-
return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
|
|
214
|
-
}
|
|
215
|
-
/**
|
|
216
|
-
* Constructs a pure integer value symbol.
|
|
407
|
+
* Recognition only. The halves are checked when the value is written.
|
|
217
408
|
*
|
|
218
|
-
* @param {
|
|
219
|
-
* @return {
|
|
409
|
+
* @param {any} value Any value.
|
|
410
|
+
* @return {boolean} True for a dual cell.
|
|
220
411
|
*/
|
|
221
|
-
static
|
|
222
|
-
return
|
|
412
|
+
static isDual(value) {
|
|
413
|
+
return asDual(value) !== null;
|
|
223
414
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
415
|
+
};
|
|
416
|
+
Object.defineProperty(exports.QvdDual.prototype, DUAL_BRAND, { value: true });
|
|
417
|
+
__name(defineHalves, "defineHalves");
|
|
418
|
+
__name(dualFromSymbol, "dualFromSymbol");
|
|
419
|
+
}
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
// src/util/readOptions.js
|
|
423
|
+
function requireRowCount(value, name, filePath) {
|
|
424
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
425
|
+
throw new exports.QvdValidationError(`${name} must be a non-negative integer`, {
|
|
426
|
+
option: name,
|
|
427
|
+
provided: value,
|
|
428
|
+
type: typeof value,
|
|
429
|
+
file: filePath
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
return value;
|
|
433
|
+
}
|
|
434
|
+
function normaliseWindow(window, filePath) {
|
|
435
|
+
if (window === null || window === void 0) {
|
|
436
|
+
return { offset: 0, limit: null };
|
|
437
|
+
}
|
|
438
|
+
if (typeof window === "number") {
|
|
439
|
+
return { offset: 0, limit: requireRowCount(window, "maxRows", filePath) };
|
|
440
|
+
}
|
|
441
|
+
if (typeof window !== "object" || Array.isArray(window)) {
|
|
442
|
+
throw new exports.QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
|
|
443
|
+
provided: window,
|
|
444
|
+
type: typeof window,
|
|
445
|
+
file: filePath
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
const { offset, limit, maxRows } = window;
|
|
449
|
+
const limitGiven = limit !== void 0 && limit !== null;
|
|
450
|
+
const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
|
|
451
|
+
if (limitGiven && maxRowsGiven) {
|
|
452
|
+
throw new exports.QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
|
|
453
|
+
maxRows,
|
|
454
|
+
limit,
|
|
455
|
+
file: filePath
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
return {
|
|
459
|
+
offset: offset === void 0 || offset === null ? 0 : requireRowCount(offset, "offset", filePath),
|
|
460
|
+
limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
function resolveWindow(window, totalRows) {
|
|
464
|
+
const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
|
|
465
|
+
const offset = Math.min(window.offset, rows);
|
|
466
|
+
return {
|
|
467
|
+
offset,
|
|
468
|
+
limit: Math.max(0, Math.min(window.limit === null ? Infinity : window.limit, rows - offset))
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
function selectFields(fields, requested, filePath) {
|
|
472
|
+
if (requested === null || requested === void 0) {
|
|
473
|
+
return fields;
|
|
474
|
+
}
|
|
475
|
+
if (!Array.isArray(requested)) {
|
|
476
|
+
throw new exports.QvdValidationError("fields must be an array of field names", {
|
|
477
|
+
provided: requested,
|
|
478
|
+
type: typeof requested,
|
|
479
|
+
file: filePath
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
const available = fields.map((field) => field["FieldName"]);
|
|
483
|
+
if (requested.length === 0) {
|
|
484
|
+
throw new exports.QvdValidationError("fields must name at least one field", {
|
|
485
|
+
availableColumns: available,
|
|
486
|
+
file: filePath
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
const seen = /* @__PURE__ */ new Set();
|
|
490
|
+
return requested.map((name) => {
|
|
491
|
+
if (typeof name !== "string") {
|
|
492
|
+
throw new exports.QvdValidationError("Field names must be strings", {
|
|
493
|
+
provided: name,
|
|
494
|
+
type: typeof name,
|
|
495
|
+
availableColumns: available,
|
|
496
|
+
file: filePath
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
if (seen.has(name)) {
|
|
500
|
+
throw new exports.QvdValidationError(`Field '${name}' is listed twice`, {
|
|
501
|
+
column: name,
|
|
502
|
+
fields: requested,
|
|
503
|
+
file: filePath
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
seen.add(name);
|
|
507
|
+
const index = available.indexOf(name);
|
|
508
|
+
if (index === -1) {
|
|
509
|
+
throw new exports.QvdValidationError(`Column '${name}' does not exist`, {
|
|
510
|
+
column: name,
|
|
511
|
+
availableColumns: available,
|
|
512
|
+
file: filePath
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
return fields[index];
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
function normaliseDuals(value, filePath) {
|
|
519
|
+
if (value === void 0 || value === null) {
|
|
520
|
+
return "number";
|
|
521
|
+
}
|
|
522
|
+
if (!DUAL_MODES.includes(value)) {
|
|
523
|
+
throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
524
|
+
option: "duals",
|
|
525
|
+
provided: value,
|
|
526
|
+
file: filePath
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
return value;
|
|
530
|
+
}
|
|
531
|
+
function normaliseCoerceNumericStrings(value, filePath) {
|
|
532
|
+
if (value === void 0 || value === null) {
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
if (typeof value !== "boolean") {
|
|
536
|
+
throw new exports.QvdValidationError("coerceNumericStrings must be true or false", {
|
|
537
|
+
option: "coerceNumericStrings",
|
|
538
|
+
provided: value,
|
|
539
|
+
type: typeof value,
|
|
540
|
+
file: filePath
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
return value;
|
|
544
|
+
}
|
|
545
|
+
function readerOptionsFrom(options) {
|
|
546
|
+
return {
|
|
547
|
+
allowedDir: options.allowedDir,
|
|
548
|
+
memorySafetyFactor: options.memorySafetyFactor,
|
|
549
|
+
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
550
|
+
fields: options.fields === void 0 ? null : options.fields,
|
|
551
|
+
duals: options.duals,
|
|
552
|
+
coerceNumericStrings: options.coerceNumericStrings,
|
|
553
|
+
onProgress: options.onProgress,
|
|
554
|
+
signal: options.signal
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
function metadataOptionsFrom(options) {
|
|
558
|
+
return {
|
|
559
|
+
allowedDir: options.allowedDir,
|
|
560
|
+
onProgress: options.onProgress,
|
|
561
|
+
signal: options.signal
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
function windowFrom(options) {
|
|
565
|
+
return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
|
|
566
|
+
}
|
|
567
|
+
var DUAL_MODES;
|
|
568
|
+
var init_readOptions = __esm({
|
|
569
|
+
"src/util/readOptions.js"() {
|
|
570
|
+
init_QvdErrors();
|
|
571
|
+
__name(requireRowCount, "requireRowCount");
|
|
572
|
+
__name(normaliseWindow, "normaliseWindow");
|
|
573
|
+
__name(resolveWindow, "resolveWindow");
|
|
574
|
+
__name(selectFields, "selectFields");
|
|
575
|
+
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
576
|
+
__name(normaliseDuals, "normaliseDuals");
|
|
577
|
+
__name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
|
|
578
|
+
__name(readerOptionsFrom, "readerOptionsFrom");
|
|
579
|
+
__name(metadataOptionsFrom, "metadataOptionsFrom");
|
|
580
|
+
__name(windowFrom, "windowFrom");
|
|
581
|
+
}
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
// src/util/storedSymbols.js
|
|
585
|
+
function trustStoredSymbols(entries) {
|
|
586
|
+
const record = Object.freeze(entries);
|
|
587
|
+
trusted.add(record);
|
|
588
|
+
return record;
|
|
589
|
+
}
|
|
590
|
+
function attachStoredSymbols(metadata, record) {
|
|
591
|
+
if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
|
|
592
|
+
Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
function refuse(message, context) {
|
|
596
|
+
throw new exports.QvdValidationError(message, context);
|
|
597
|
+
}
|
|
598
|
+
function normaliseStoredSymbols(record) {
|
|
599
|
+
if (record === null || record === void 0) {
|
|
600
|
+
return null;
|
|
601
|
+
}
|
|
602
|
+
if (typeof record === "object" && trusted.has(record)) {
|
|
603
|
+
return record;
|
|
604
|
+
}
|
|
605
|
+
if (!Array.isArray(record)) {
|
|
606
|
+
refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
|
|
607
|
+
option: "storedSymbols",
|
|
608
|
+
type: typeof record
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
const fields = /* @__PURE__ */ new Set();
|
|
612
|
+
const entries = record.map((entry, index) => {
|
|
613
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
|
|
614
|
+
refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
|
|
615
|
+
}
|
|
616
|
+
const { field, values, numbers, texts } = entry;
|
|
617
|
+
if (fields.has(field)) {
|
|
618
|
+
refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
|
|
619
|
+
}
|
|
620
|
+
fields.add(field);
|
|
621
|
+
if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
|
|
622
|
+
refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
|
|
623
|
+
field,
|
|
624
|
+
entry: index
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
for (let symbol = 0; symbol < values.length; symbol++) {
|
|
628
|
+
const value = values[symbol];
|
|
629
|
+
const number = numbers[symbol];
|
|
630
|
+
const text = texts[symbol];
|
|
631
|
+
const context = { field, symbol };
|
|
632
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
633
|
+
refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
|
|
634
|
+
...context,
|
|
635
|
+
type: typeof value
|
|
636
|
+
});
|
|
232
637
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
*
|
|
236
|
-
* @param {string} stringValue The string value.
|
|
237
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
238
|
-
*/
|
|
239
|
-
static fromStringValue(stringValue) {
|
|
240
|
-
return new _QvdSymbol(null, null, stringValue);
|
|
638
|
+
if (number !== null) {
|
|
639
|
+
checkNumber(number, "A stored symbol's number", context);
|
|
241
640
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
*
|
|
245
|
-
* @param {number} intValue The integer value.
|
|
246
|
-
* @param {string} stringValue The string value.
|
|
247
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
248
|
-
*/
|
|
249
|
-
static fromDualIntValue(intValue, stringValue) {
|
|
250
|
-
return new _QvdSymbol(intValue, null, stringValue);
|
|
641
|
+
if (text !== null) {
|
|
642
|
+
checkText(text, "A stored symbol's text", context);
|
|
251
643
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
*/
|
|
259
|
-
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
260
|
-
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
644
|
+
const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
|
|
645
|
+
if (!consistent) {
|
|
646
|
+
refuse(
|
|
647
|
+
number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
|
|
648
|
+
{ ...context, value, number, text }
|
|
649
|
+
);
|
|
261
650
|
}
|
|
262
|
-
}
|
|
651
|
+
}
|
|
652
|
+
return Object.freeze({
|
|
653
|
+
field,
|
|
654
|
+
values: Object.freeze(values.slice()),
|
|
655
|
+
numbers: Object.freeze(numbers.slice()),
|
|
656
|
+
texts: Object.freeze(texts.slice())
|
|
657
|
+
});
|
|
658
|
+
});
|
|
659
|
+
return trustStoredSymbols(entries);
|
|
660
|
+
}
|
|
661
|
+
function narrowStoredSymbols(record, columns) {
|
|
662
|
+
if (record === null) {
|
|
663
|
+
return null;
|
|
664
|
+
}
|
|
665
|
+
const kept = record.filter((entry) => columns.includes(entry.field));
|
|
666
|
+
return kept.length === record.length ? record : trustStoredSymbols(kept);
|
|
667
|
+
}
|
|
668
|
+
function storedSymbolsEntry(record, field) {
|
|
669
|
+
if (record === null) {
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
return record.find((entry) => entry.field === field) ?? null;
|
|
673
|
+
}
|
|
674
|
+
function firstTextByValue(entry) {
|
|
675
|
+
const byValue = /* @__PURE__ */ new Map();
|
|
676
|
+
for (let index = entry.values.length - 1; index >= 0; index--) {
|
|
677
|
+
const text = entry.texts[index];
|
|
678
|
+
if (text !== null) {
|
|
679
|
+
byValue.set(entry.values[index], text);
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
return byValue;
|
|
683
|
+
}
|
|
684
|
+
function storedTextOf(entry, value) {
|
|
685
|
+
let byValue = firstTexts.get(entry);
|
|
686
|
+
if (byValue === void 0) {
|
|
687
|
+
byValue = firstTextByValue(entry);
|
|
688
|
+
firstTexts.set(entry, byValue);
|
|
689
|
+
}
|
|
690
|
+
return byValue.get(value) ?? null;
|
|
691
|
+
}
|
|
692
|
+
function sameValueZero(a, b) {
|
|
693
|
+
return a === b || a !== a && b !== b;
|
|
694
|
+
}
|
|
695
|
+
var STORED_SYMBOLS, trusted, firstTexts;
|
|
696
|
+
var init_storedSymbols = __esm({
|
|
697
|
+
"src/util/storedSymbols.js"() {
|
|
698
|
+
init_QvdErrors();
|
|
699
|
+
init_cellRules();
|
|
700
|
+
STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
|
|
701
|
+
trusted = /* @__PURE__ */ new WeakSet();
|
|
702
|
+
__name(trustStoredSymbols, "trustStoredSymbols");
|
|
703
|
+
__name(attachStoredSymbols, "attachStoredSymbols");
|
|
704
|
+
__name(refuse, "refuse");
|
|
705
|
+
__name(normaliseStoredSymbols, "normaliseStoredSymbols");
|
|
706
|
+
__name(narrowStoredSymbols, "narrowStoredSymbols");
|
|
707
|
+
__name(storedSymbolsEntry, "storedSymbolsEntry");
|
|
708
|
+
firstTexts = /* @__PURE__ */ new WeakMap();
|
|
709
|
+
__name(firstTextByValue, "firstTextByValue");
|
|
710
|
+
__name(storedTextOf, "storedTextOf");
|
|
711
|
+
__name(sameValueZero, "sameValueZero");
|
|
263
712
|
}
|
|
264
713
|
});
|
|
265
714
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
@@ -364,6 +813,10 @@ function validatePath(filePath, allowedDir) {
|
|
|
364
813
|
var init_validatePath = __esm({
|
|
365
814
|
"src/util/validatePath.js"() {
|
|
366
815
|
init_QvdErrors();
|
|
816
|
+
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
817
|
+
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
818
|
+
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
819
|
+
__name(validatePath, "validatePath");
|
|
367
820
|
}
|
|
368
821
|
});
|
|
369
822
|
|
|
@@ -413,6 +866,9 @@ var init_bitUtils = __esm({
|
|
|
413
866
|
"src/util/bitUtils.js"() {
|
|
414
867
|
MAX_BIT_WIDTH = 31;
|
|
415
868
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
869
|
+
__name(fieldGeometry, "fieldGeometry");
|
|
870
|
+
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
871
|
+
__name(writeBitField, "writeBitField");
|
|
416
872
|
}
|
|
417
873
|
});
|
|
418
874
|
|
|
@@ -421,14 +877,310 @@ var QvdFileWriter_exports = {};
|
|
|
421
877
|
__export(QvdFileWriter_exports, {
|
|
422
878
|
QvdFileWriter: () => exports.QvdFileWriter
|
|
423
879
|
});
|
|
424
|
-
|
|
880
|
+
function notXmlIndex(value) {
|
|
881
|
+
for (let index = 0; index < value.length; index++) {
|
|
882
|
+
const unit = value.charCodeAt(index);
|
|
883
|
+
if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
|
|
884
|
+
return index;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
return -1;
|
|
888
|
+
}
|
|
889
|
+
function checkHeaderText(value, subject, context) {
|
|
890
|
+
checkText(value, subject, context);
|
|
891
|
+
const position = notXmlIndex(value);
|
|
892
|
+
if (position !== -1) {
|
|
893
|
+
const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
|
|
894
|
+
throw new exports.QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
function checkHeaderTexts(value, property, owner, context) {
|
|
898
|
+
if (typeof value === "string") {
|
|
899
|
+
checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
|
|
900
|
+
} else if (Array.isArray(value)) {
|
|
901
|
+
value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
|
|
902
|
+
} else if (value !== null && typeof value === "object") {
|
|
903
|
+
for (const [key, item] of Object.entries(value)) {
|
|
904
|
+
checkHeaderTexts(item, `${property}.${key}`, owner, context);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
function validateColumnNames(columns, filePath) {
|
|
909
|
+
const seen = /* @__PURE__ */ new Set();
|
|
910
|
+
columns.forEach((name, index) => {
|
|
911
|
+
if (typeof name !== "string" || name.length === 0) {
|
|
912
|
+
throw new exports.QvdValidationError("Field names must be non-empty strings", {
|
|
913
|
+
column: index,
|
|
914
|
+
provided: name,
|
|
915
|
+
type: typeof name,
|
|
916
|
+
file: filePath,
|
|
917
|
+
stage: "buildSymbolTable"
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
|
|
921
|
+
if (seen.has(name)) {
|
|
922
|
+
throw new exports.QvdValidationError(`Field '${name}' appears twice`, {
|
|
923
|
+
column: name,
|
|
924
|
+
columnIndex: index,
|
|
925
|
+
file: filePath,
|
|
926
|
+
stage: "buildSymbolTable"
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
seen.add(name);
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
function refuseCell(value, column, row, filePath) {
|
|
933
|
+
let resemblesDual = false;
|
|
934
|
+
try {
|
|
935
|
+
resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
|
|
936
|
+
} catch {
|
|
937
|
+
}
|
|
938
|
+
throw new exports.QvdValidationError(
|
|
939
|
+
`A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
|
|
940
|
+
{
|
|
941
|
+
column,
|
|
942
|
+
row,
|
|
943
|
+
type: typeof value,
|
|
944
|
+
constructor: constructorName(value) ?? void 0,
|
|
945
|
+
file: filePath,
|
|
946
|
+
stage: "buildSymbolTable"
|
|
947
|
+
}
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
function valuesAreNumbers(entry) {
|
|
951
|
+
const { values, numbers } = entry;
|
|
952
|
+
for (let index = 0; index < values.length; index++) {
|
|
953
|
+
if (!sameValueZero(values[index], numbers[index])) {
|
|
954
|
+
return false;
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return true;
|
|
958
|
+
}
|
|
959
|
+
function newSlot(column, key, text) {
|
|
960
|
+
const slot = column.keys.length;
|
|
961
|
+
column.keys.push(key);
|
|
962
|
+
column.texts.push(text);
|
|
963
|
+
if (typeof key === "number") {
|
|
964
|
+
column.facts.hasNumber = true;
|
|
965
|
+
if (!Number.isInteger(key)) column.facts.hasFraction = true;
|
|
966
|
+
} else {
|
|
967
|
+
column.facts.hasNonNumber = true;
|
|
968
|
+
}
|
|
969
|
+
return slot;
|
|
970
|
+
}
|
|
971
|
+
function slotFor(column, key, text) {
|
|
972
|
+
let slot = column.byKey.get(key);
|
|
973
|
+
if (slot === void 0) {
|
|
974
|
+
slot = newSlot(column, key, text);
|
|
975
|
+
column.byKey.set(key, slot);
|
|
976
|
+
} else if (text !== null && column.texts[slot] === null && typeof key === "number") {
|
|
977
|
+
column.texts[slot] = text;
|
|
978
|
+
}
|
|
979
|
+
return slot;
|
|
980
|
+
}
|
|
981
|
+
function standsForSeveral(indices, numbers, texts) {
|
|
982
|
+
let number = null;
|
|
983
|
+
let string = null;
|
|
984
|
+
for (const index of indices) {
|
|
985
|
+
if (numbers[index] !== null) {
|
|
986
|
+
if (number === null) {
|
|
987
|
+
number = numbers[index];
|
|
988
|
+
} else if (!sameValueZero(number, numbers[index])) {
|
|
989
|
+
return true;
|
|
990
|
+
}
|
|
991
|
+
} else if (string === null) {
|
|
992
|
+
string = texts[index];
|
|
993
|
+
} else if (string !== texts[index]) {
|
|
994
|
+
return true;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
return number !== null && string !== null;
|
|
998
|
+
}
|
|
999
|
+
function ambiguousAsUncoercedText(entry) {
|
|
1000
|
+
const { values, numbers, texts } = entry;
|
|
1001
|
+
for (let index = 0; index < values.length; index++) {
|
|
1002
|
+
if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
|
|
1003
|
+
if (!isNumericText(texts[index])) {
|
|
1004
|
+
return false;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
const byShown = /* @__PURE__ */ new Map();
|
|
1009
|
+
for (let index = 0; index < values.length; index++) {
|
|
1010
|
+
const shown = texts[index] ?? numbers[index];
|
|
1011
|
+
const indices = byShown.get(shown);
|
|
1012
|
+
if (indices === void 0) {
|
|
1013
|
+
byShown.set(shown, [index]);
|
|
1014
|
+
} else {
|
|
1015
|
+
indices.push(index);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
for (const indices of byShown.values()) {
|
|
1019
|
+
if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
|
|
1020
|
+
return true;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
return false;
|
|
1024
|
+
}
|
|
1025
|
+
function unambiguousWithoutDuals(column) {
|
|
1026
|
+
const { numbers, texts } = column.entry;
|
|
1027
|
+
for (const found of column.byValue.values()) {
|
|
1028
|
+
if (typeof found !== "number") {
|
|
1029
|
+
const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
|
|
1030
|
+
if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
|
|
1031
|
+
return false;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
return true;
|
|
1036
|
+
}
|
|
1037
|
+
function ambiguityRemedy(column) {
|
|
1038
|
+
const { values, numbers, texts } = column.entry;
|
|
1039
|
+
let coerced = false;
|
|
1040
|
+
let other = false;
|
|
1041
|
+
for (const [value, found] of column.byValue) {
|
|
1042
|
+
if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
|
|
1043
|
+
if (typeof value === "number") {
|
|
1044
|
+
coerced = true;
|
|
1045
|
+
} else {
|
|
1046
|
+
other = true;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
if (!coerced) {
|
|
1051
|
+
return "Read the field with {duals: 'both'}, or write QvdDual cells.";
|
|
1052
|
+
}
|
|
1053
|
+
if (!other && !ambiguousAsUncoercedText(column.entry)) {
|
|
1054
|
+
return "Read the field without {coerceNumericStrings: true}.";
|
|
1055
|
+
}
|
|
1056
|
+
if (unambiguousWithoutDuals(column)) {
|
|
1057
|
+
return "Read the field with {duals: 'both'}.";
|
|
1058
|
+
}
|
|
1059
|
+
return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
|
|
1060
|
+
}
|
|
1061
|
+
function slotForCell(column, value, row, filePath) {
|
|
1062
|
+
if (column.byCell === column.byKey) {
|
|
1063
|
+
return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
|
|
1064
|
+
}
|
|
1065
|
+
const found = column.byValue.get(value);
|
|
1066
|
+
if (found === void 0) {
|
|
1067
|
+
return slotFor(column, value, null);
|
|
1068
|
+
}
|
|
1069
|
+
const { numbers, texts } = column.entry;
|
|
1070
|
+
const indices = typeof found === "number" ? [found] : found;
|
|
1071
|
+
if (standsForSeveral(indices, numbers, texts)) {
|
|
1072
|
+
const stored = [];
|
|
1073
|
+
for (const index of indices) {
|
|
1074
|
+
if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
|
|
1075
|
+
stored.push({ number: numbers[index], text: texts[index] });
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
throw new exports.QvdValidationError(
|
|
1079
|
+
`The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
|
|
1080
|
+
{ column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
let number = null;
|
|
1084
|
+
let numberText = null;
|
|
1085
|
+
for (const index of indices) {
|
|
1086
|
+
if (numbers[index] === null) {
|
|
1087
|
+
return slotFor(column, texts[index], null);
|
|
1088
|
+
}
|
|
1089
|
+
number ??= numbers[index];
|
|
1090
|
+
numberText ??= texts[index];
|
|
1091
|
+
}
|
|
1092
|
+
return slotFor(column, number, numberText);
|
|
1093
|
+
}
|
|
1094
|
+
function slotForObject(column, value, row, filePath) {
|
|
1095
|
+
const dual = asDual(value);
|
|
1096
|
+
if (dual === null) {
|
|
1097
|
+
refuseCell(value, column.name, row, filePath);
|
|
1098
|
+
}
|
|
1099
|
+
const { number, text } = dual;
|
|
1100
|
+
if (numberProblem(number) !== null) {
|
|
1101
|
+
checkNumber(number, "The number of a dual value", {
|
|
1102
|
+
column: column.name,
|
|
1103
|
+
row,
|
|
1104
|
+
half: "number",
|
|
1105
|
+
file: filePath,
|
|
1106
|
+
stage: "buildSymbolTable"
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
if (textProblem(text) !== null) {
|
|
1110
|
+
checkText(text, "The text of a dual value", {
|
|
1111
|
+
column: column.name,
|
|
1112
|
+
row,
|
|
1113
|
+
half: "text",
|
|
1114
|
+
file: filePath,
|
|
1115
|
+
stage: "buildSymbolTable"
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
const slot = slotFor(column, number, text);
|
|
1119
|
+
if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
|
|
1120
|
+
column.byObject.set(value, slot);
|
|
1121
|
+
}
|
|
1122
|
+
return slot;
|
|
1123
|
+
}
|
|
1124
|
+
function pruneContradictedTags(tags, facts) {
|
|
1125
|
+
if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
|
|
1126
|
+
return tags || {};
|
|
1127
|
+
}
|
|
1128
|
+
const list = Array.isArray(tags.String) ? tags.String : [tags.String];
|
|
1129
|
+
const kept = list.filter((tag) => {
|
|
1130
|
+
if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
|
|
1131
|
+
if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
|
|
1132
|
+
if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
|
|
1133
|
+
return true;
|
|
1134
|
+
});
|
|
1135
|
+
if (kept.length === list.length) {
|
|
1136
|
+
return tags;
|
|
1137
|
+
}
|
|
1138
|
+
return kept.length === 0 ? {} : { ...tags, String: kept };
|
|
1139
|
+
}
|
|
1140
|
+
function resetContradictedNumberFormat(numberFormat, facts) {
|
|
1141
|
+
if (!numberFormat) {
|
|
1142
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1143
|
+
}
|
|
1144
|
+
if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
|
|
1145
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1146
|
+
}
|
|
1147
|
+
return numberFormat;
|
|
1148
|
+
}
|
|
1149
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT; exports.QvdFileWriter = void 0;
|
|
425
1150
|
var init_QvdFileWriter = __esm({
|
|
426
1151
|
"src/QvdFileWriter.js"() {
|
|
427
|
-
init_QvdSymbol();
|
|
428
1152
|
init_QvdErrors();
|
|
429
1153
|
init_validatePath();
|
|
430
1154
|
init_bitUtils();
|
|
431
|
-
|
|
1155
|
+
init_cellRules();
|
|
1156
|
+
init_symbolBytes();
|
|
1157
|
+
init_storedSymbols();
|
|
1158
|
+
__name(notXmlIndex, "notXmlIndex");
|
|
1159
|
+
__name(checkHeaderText, "checkHeaderText");
|
|
1160
|
+
__name(checkHeaderTexts, "checkHeaderTexts");
|
|
1161
|
+
__name(validateColumnNames, "validateColumnNames");
|
|
1162
|
+
__name(refuseCell, "refuseCell");
|
|
1163
|
+
OBJECT_MEMO_BASE = 64;
|
|
1164
|
+
__name(valuesAreNumbers, "valuesAreNumbers");
|
|
1165
|
+
__name(newSlot, "newSlot");
|
|
1166
|
+
__name(slotFor, "slotFor");
|
|
1167
|
+
__name(standsForSeveral, "standsForSeveral");
|
|
1168
|
+
__name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
|
|
1169
|
+
__name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
|
|
1170
|
+
__name(ambiguityRemedy, "ambiguityRemedy");
|
|
1171
|
+
__name(slotForCell, "slotForCell");
|
|
1172
|
+
__name(slotForObject, "slotForObject");
|
|
1173
|
+
NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
|
|
1174
|
+
TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
|
|
1175
|
+
WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
|
|
1176
|
+
NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
|
|
1177
|
+
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1178
|
+
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1179
|
+
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1180
|
+
exports.QvdFileWriter = class {
|
|
1181
|
+
static {
|
|
1182
|
+
__name(this, "QvdFileWriter");
|
|
1183
|
+
}
|
|
432
1184
|
/**
|
|
433
1185
|
* Constructs a new QVD file writer.
|
|
434
1186
|
*
|
|
@@ -449,7 +1201,8 @@ var init_QvdFileWriter = __esm({
|
|
|
449
1201
|
this._onProgress = onProgress;
|
|
450
1202
|
this._header = null;
|
|
451
1203
|
this._symbolBuffer = null;
|
|
452
|
-
this.
|
|
1204
|
+
this._symbolCounts = null;
|
|
1205
|
+
this._symbolFacts = null;
|
|
453
1206
|
this._symbolTableMetadata = null;
|
|
454
1207
|
this._indexBuffer = null;
|
|
455
1208
|
this._symbolIndexByValue = null;
|
|
@@ -560,19 +1313,12 @@ var init_QvdFileWriter = __esm({
|
|
|
560
1313
|
BitOffset: this._indexTableMetadata?.[index][0],
|
|
561
1314
|
BitWidth: this._indexTableMetadata?.[index][1],
|
|
562
1315
|
Bias: this._indexTableMetadata?.[index][2],
|
|
563
|
-
NoOfSymbols: this.
|
|
1316
|
+
NoOfSymbols: this._symbolCounts?.[index],
|
|
564
1317
|
Offset: this._symbolTableMetadata?.[index][0],
|
|
565
1318
|
Length: this._symbolTableMetadata?.[index][1],
|
|
566
1319
|
Comment: existingField?.Comment || "",
|
|
567
|
-
NumberFormat: existingField?.NumberFormat
|
|
568
|
-
|
|
569
|
-
nDec: "0",
|
|
570
|
-
UseThou: "0",
|
|
571
|
-
Fmt: "",
|
|
572
|
-
Dec: "",
|
|
573
|
-
Thou: ""
|
|
574
|
-
},
|
|
575
|
-
Tags: existingField?.Tags || {}
|
|
1320
|
+
NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
|
|
1321
|
+
Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
|
|
576
1322
|
};
|
|
577
1323
|
})
|
|
578
1324
|
},
|
|
@@ -582,6 +1328,19 @@ var init_QvdFileWriter = __esm({
|
|
|
582
1328
|
Length: this._indexBuffer?.length
|
|
583
1329
|
}
|
|
584
1330
|
};
|
|
1331
|
+
const { Fields, ...table } = xmlObject.QvdTableHeader;
|
|
1332
|
+
for (const [property, value] of Object.entries(table)) {
|
|
1333
|
+
checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
|
|
1334
|
+
}
|
|
1335
|
+
for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
|
|
1336
|
+
for (const [property, value] of Object.entries(field)) {
|
|
1337
|
+
checkHeaderTexts(value, property, `field '${FieldName}'`, {
|
|
1338
|
+
column: FieldName,
|
|
1339
|
+
file: this._path,
|
|
1340
|
+
stage: "buildHeader"
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
585
1344
|
const builder = new xml2__default.default.Builder({
|
|
586
1345
|
renderOpts: {
|
|
587
1346
|
pretty: true,
|
|
@@ -595,25 +1354,49 @@ var init_QvdFileWriter = __esm({
|
|
|
595
1354
|
/**
|
|
596
1355
|
* Builds the symbol table of the QVD file.
|
|
597
1356
|
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
600
|
-
*
|
|
1357
|
+
* One pass over the rows finds each column's distinct values in the order they first appear,
|
|
1358
|
+
* which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
|
|
1359
|
+
* column encodes them: every symbol is sized first, then written into one buffer of exactly that
|
|
1360
|
+
* size.
|
|
1361
|
+
*
|
|
1362
|
+
* What each value is stored as:
|
|
1363
|
+
*
|
|
1364
|
+
* | Cell | Symbol |
|
|
1365
|
+
* | --- | --- |
|
|
1366
|
+
* | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
|
|
1367
|
+
* | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
|
|
1368
|
+
* | any other finite number | pure double, type 2 |
|
|
1369
|
+
* | a string | pure string, type 4 |
|
|
1370
|
+
* | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
|
|
1371
|
+
* | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
|
|
1372
|
+
*
|
|
1373
|
+
* A number is a pure number, with no text. It used to be written as a dual whose text was
|
|
1374
|
+
* `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
|
|
1375
|
+
* file Qlik wrote with pure numbers came back out of a read and a write with every one of them
|
|
1376
|
+
* turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
|
|
1377
|
+
* A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
|
|
1378
|
+
* UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
|
|
1379
|
+
* an unpaired surrogate has no UTF-8 encoding at all.
|
|
601
1380
|
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
* 4. Serialize symbols to binary format and update metadata
|
|
1381
|
+
* A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
|
|
1382
|
+
* symbol with the first text in row order, and a plain number with the same number as a dual joins
|
|
1383
|
+
* the dual, in either order - so no text a caller supplied is lost to a plain number that came
|
|
1384
|
+
* first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
|
|
607
1385
|
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
* -
|
|
611
|
-
*
|
|
1386
|
+
* A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
|
|
1387
|
+
* string read as a number. Its record says what each such cell stands for, so the frame writes back
|
|
1388
|
+
* the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
|
|
1389
|
+
* rows the cells were moved to. A cell the record maps to more than one stored value is refused.
|
|
1390
|
+
*
|
|
1391
|
+
* The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
|
|
1392
|
+
* for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
|
|
1393
|
+
* repository; `benchmarks/` measures what the writer costs now, which is the useful number.
|
|
612
1394
|
*
|
|
613
1395
|
* @private
|
|
614
1396
|
*/
|
|
615
1397
|
_buildSymbolTable() {
|
|
616
|
-
this.
|
|
1398
|
+
this._symbolCounts = [];
|
|
1399
|
+
this._symbolFacts = [];
|
|
617
1400
|
this._symbolTableMetadata = [];
|
|
618
1401
|
this._symbolIndexByValue = [];
|
|
619
1402
|
if (this._df.columns.length === 0) {
|
|
@@ -626,33 +1409,144 @@ var init_QvdFileWriter = __esm({
|
|
|
626
1409
|
const data = this._df.data;
|
|
627
1410
|
const numColumns = columns.length;
|
|
628
1411
|
const numRows = data.length;
|
|
1412
|
+
validateColumnNames(columns, this._path);
|
|
1413
|
+
const record = normaliseStoredSymbols(this._df.storedSymbols);
|
|
629
1414
|
this._emitProgress("symbol-table", 0, numColumns);
|
|
630
|
-
const
|
|
1415
|
+
const state = columns.map((name) => {
|
|
1416
|
+
const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
|
|
1417
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1418
|
+
let byValue = null;
|
|
1419
|
+
let textByNumber = null;
|
|
1420
|
+
if (entry !== null && valuesAreNumbers(entry)) {
|
|
1421
|
+
textByNumber = firstTextByValue(entry);
|
|
1422
|
+
} else if (entry !== null) {
|
|
1423
|
+
byValue = /* @__PURE__ */ new Map();
|
|
1424
|
+
for (let index = 0; index < entry.values.length; index++) {
|
|
1425
|
+
const found = byValue.get(entry.values[index]);
|
|
1426
|
+
if (found === void 0) {
|
|
1427
|
+
byValue.set(entry.values[index], index);
|
|
1428
|
+
} else if (typeof found === "number") {
|
|
1429
|
+
byValue.set(entry.values[index], [found, index]);
|
|
1430
|
+
} else {
|
|
1431
|
+
found.push(index);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
return {
|
|
1436
|
+
name,
|
|
1437
|
+
keys: [],
|
|
1438
|
+
texts: [],
|
|
1439
|
+
byKey,
|
|
1440
|
+
// A cell is its own key without a record entry, and with one whose values are their numbers, so
|
|
1441
|
+
// one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
|
|
1442
|
+
byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
|
|
1443
|
+
byObject: /* @__PURE__ */ new Map(),
|
|
1444
|
+
entry,
|
|
1445
|
+
textByNumber,
|
|
1446
|
+
byValue,
|
|
1447
|
+
facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
|
|
1448
|
+
};
|
|
1449
|
+
});
|
|
1450
|
+
const byCells = state.map((column) => column.byCell);
|
|
1451
|
+
const byObjects = state.map((column) => column.byObject);
|
|
631
1452
|
const containsNull = columns.map(() => false);
|
|
632
1453
|
for (let row = 0; row < numRows; row++) {
|
|
633
1454
|
const values = data[row];
|
|
1455
|
+
if (values !== null && values !== void 0 && !Array.isArray(values)) {
|
|
1456
|
+
throw new exports.QvdValidationError("Each row must be an array of values", {
|
|
1457
|
+
row,
|
|
1458
|
+
type: typeof values,
|
|
1459
|
+
file: this._path,
|
|
1460
|
+
stage: "buildSymbolTable"
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
if (values !== null && values !== void 0 && values.length > numColumns) {
|
|
1464
|
+
throw new exports.QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
|
|
1465
|
+
row,
|
|
1466
|
+
values: values.length,
|
|
1467
|
+
fields: numColumns,
|
|
1468
|
+
file: this._path,
|
|
1469
|
+
stage: "buildSymbolTable"
|
|
1470
|
+
});
|
|
1471
|
+
}
|
|
634
1472
|
for (let column = 0; column < numColumns; column++) {
|
|
635
1473
|
const value = values?.[column];
|
|
636
1474
|
if (value === null || value === void 0) {
|
|
637
1475
|
containsNull[column] = true;
|
|
638
1476
|
continue;
|
|
639
1477
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
1478
|
+
if (typeof value === "object") {
|
|
1479
|
+
if (!byObjects[column].has(value)) {
|
|
1480
|
+
slotForObject(state[column], value, row, this._path);
|
|
1481
|
+
}
|
|
1482
|
+
continue;
|
|
1483
|
+
}
|
|
1484
|
+
if (byCells[column].has(value)) {
|
|
1485
|
+
continue;
|
|
1486
|
+
}
|
|
1487
|
+
if (typeof value === "number") {
|
|
1488
|
+
if (numberProblem(value) !== null) {
|
|
1489
|
+
checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
|
|
1490
|
+
}
|
|
1491
|
+
} else if (typeof value === "string") {
|
|
1492
|
+
if (textProblem(value) !== null) {
|
|
1493
|
+
checkText(value, "A string value", {
|
|
1494
|
+
column: columns[column],
|
|
1495
|
+
row,
|
|
1496
|
+
file: this._path,
|
|
1497
|
+
stage: "buildSymbolTable"
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
} else {
|
|
1501
|
+
refuseCell(value, columns[column], row, this._path);
|
|
643
1502
|
}
|
|
1503
|
+
byCells[column].set(value, slotForCell(state[column], value, row, this._path));
|
|
644
1504
|
}
|
|
645
1505
|
}
|
|
646
1506
|
const columnBuffers = [];
|
|
647
1507
|
let symbolsOffset = 0;
|
|
648
1508
|
for (let column = 0; column < numColumns; column++) {
|
|
649
|
-
const
|
|
650
|
-
const
|
|
1509
|
+
const { keys, texts } = state[column];
|
|
1510
|
+
const kinds = new Uint8Array(keys.length);
|
|
1511
|
+
let byteLength = 0;
|
|
1512
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1513
|
+
const key = keys[slot];
|
|
1514
|
+
const text = texts[slot];
|
|
1515
|
+
if (typeof key === "number") {
|
|
1516
|
+
if (numberProblem(key) !== null) {
|
|
1517
|
+
checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
|
|
1518
|
+
}
|
|
1519
|
+
if (text !== null && textProblem(text) !== null) {
|
|
1520
|
+
checkText(text, "The text of a dual value", {
|
|
1521
|
+
column: columns[column],
|
|
1522
|
+
file: this._path,
|
|
1523
|
+
stage: "buildSymbolTable"
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1526
|
+
kinds[slot] = kindOf(key, text);
|
|
1527
|
+
byteLength += symbolByteLength(kinds[slot], key, text);
|
|
1528
|
+
} else {
|
|
1529
|
+
kinds[slot] = kindOf(null, key);
|
|
1530
|
+
byteLength += symbolByteLength(kinds[slot], null, key);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
const columnBuffer = Buffer.allocUnsafe(byteLength);
|
|
1534
|
+
let offset = 0;
|
|
1535
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1536
|
+
const key = keys[slot];
|
|
1537
|
+
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1538
|
+
}
|
|
1539
|
+
assert2__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
651
1540
|
columnBuffers.push(columnBuffer);
|
|
652
|
-
this._symbolTableMetadata?.push([symbolsOffset,
|
|
653
|
-
this.
|
|
654
|
-
this.
|
|
655
|
-
|
|
1541
|
+
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1542
|
+
this._symbolCounts?.push(keys.length);
|
|
1543
|
+
this._symbolFacts?.push(state[column].facts);
|
|
1544
|
+
this._symbolIndexByValue?.push({
|
|
1545
|
+
byCell: state[column].byCell,
|
|
1546
|
+
byKey: state[column].byKey,
|
|
1547
|
+
byObject: state[column].byObject
|
|
1548
|
+
});
|
|
1549
|
+
symbolsOffset += byteLength;
|
|
656
1550
|
this._emitProgress("symbol-table", column + 1, numColumns);
|
|
657
1551
|
}
|
|
658
1552
|
this._symbolBuffer = Buffer.concat(columnBuffers);
|
|
@@ -681,7 +1575,7 @@ var init_QvdFileWriter = __esm({
|
|
|
681
1575
|
* @private
|
|
682
1576
|
*/
|
|
683
1577
|
_buildIndexTable() {
|
|
684
|
-
assert2__default.default(this.
|
|
1578
|
+
assert2__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
685
1579
|
assert2__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
686
1580
|
assert2__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
687
1581
|
this._indexTableMetadata = [];
|
|
@@ -695,20 +1589,24 @@ var init_QvdFileWriter = __esm({
|
|
|
695
1589
|
let totalBits = 0;
|
|
696
1590
|
for (let column = 0; column < numColumns; column++) {
|
|
697
1591
|
const fieldContainsNull = this._symbolTableMetadata[column][2];
|
|
698
|
-
const symbolCount = this.
|
|
1592
|
+
const symbolCount = this._symbolCounts[column];
|
|
699
1593
|
const nullShift = fieldContainsNull ? 2 : 0;
|
|
700
1594
|
const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
|
|
701
1595
|
const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
1596
|
+
const lookup = this._symbolIndexByValue[column];
|
|
1597
|
+
const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
|
|
1598
|
+
for (const map of maps) {
|
|
1599
|
+
for (const index of map.values()) {
|
|
1600
|
+
if (index + nullShift > maxStoredIndex) {
|
|
1601
|
+
throw new exports.QvdValidationError("The symbol table and the index table are out of sync", {
|
|
1602
|
+
field: columns[column],
|
|
1603
|
+
storedIndex: index + nullShift,
|
|
1604
|
+
maxStoredIndex,
|
|
1605
|
+
symbolCount,
|
|
1606
|
+
file: this._path,
|
|
1607
|
+
stage: "buildIndexTable"
|
|
1608
|
+
});
|
|
1609
|
+
}
|
|
712
1610
|
}
|
|
713
1611
|
}
|
|
714
1612
|
layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
|
|
@@ -719,6 +1617,9 @@ var init_QvdFileWriter = __esm({
|
|
|
719
1617
|
this._recordByteSize = recordByteSize;
|
|
720
1618
|
this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
|
|
721
1619
|
const progressInterval = Math.max(1, Math.floor(numRows / 100));
|
|
1620
|
+
const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
|
|
1621
|
+
const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
|
|
1622
|
+
const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
|
|
722
1623
|
for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
|
|
723
1624
|
const values = data[row];
|
|
724
1625
|
for (let column = 0; column < numColumns; column++) {
|
|
@@ -726,7 +1627,7 @@ var init_QvdFileWriter = __esm({
|
|
|
726
1627
|
if (value === null || value === void 0) {
|
|
727
1628
|
continue;
|
|
728
1629
|
}
|
|
729
|
-
const index =
|
|
1630
|
+
const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
|
|
730
1631
|
if (index === void 0) {
|
|
731
1632
|
throw new exports.QvdValidationError("A value is missing from the symbol table", {
|
|
732
1633
|
field: columns[column],
|
|
@@ -743,29 +1644,6 @@ var init_QvdFileWriter = __esm({
|
|
|
743
1644
|
}
|
|
744
1645
|
this._symbolIndexByValue = null;
|
|
745
1646
|
}
|
|
746
|
-
/**
|
|
747
|
-
* Converts a raw value/literal to a QVD symbol.
|
|
748
|
-
*
|
|
749
|
-
* @param {any} raw The raw value/literal to convert.
|
|
750
|
-
* @return {QvdSymbol|null} The converted QVD symbol.
|
|
751
|
-
*/
|
|
752
|
-
static _convertRawToSymbol(raw) {
|
|
753
|
-
if (raw === null || raw === void 0) {
|
|
754
|
-
return null;
|
|
755
|
-
}
|
|
756
|
-
const INT32_MIN = -2147483648;
|
|
757
|
-
const INT32_MAX = 2147483647;
|
|
758
|
-
const isInteger = typeof raw === "number" && Number.isInteger(raw);
|
|
759
|
-
const isFloat = typeof raw === "number" && !Number.isInteger(raw);
|
|
760
|
-
const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
|
|
761
|
-
if (isInteger && isWithinInt32Range) {
|
|
762
|
-
return exports.QvdSymbol.fromDualIntValue(raw, raw.toString());
|
|
763
|
-
} else if (isFloat || isInteger && !isWithinInt32Range) {
|
|
764
|
-
return exports.QvdSymbol.fromDualDoubleValue(raw, raw.toString());
|
|
765
|
-
} else {
|
|
766
|
-
return exports.QvdSymbol.fromStringValue(raw);
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
1647
|
/**
|
|
770
1648
|
* Persists the data frame to a QVD file.
|
|
771
1649
|
*/
|
|
@@ -819,11 +1697,12 @@ function estimateRowMemory(rows, columnCount) {
|
|
|
819
1697
|
}
|
|
820
1698
|
return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
|
|
821
1699
|
}
|
|
822
|
-
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
1700
|
+
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
|
|
823
1701
|
const FULL_PARSE_OVERHEAD = 6;
|
|
824
1702
|
const MINIMAL_OVERHEAD = 0.01;
|
|
825
1703
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
826
|
-
const
|
|
1704
|
+
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
1705
|
+
const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
|
|
827
1706
|
if (maxRows === null || maxRows >= totalRows) {
|
|
828
1707
|
return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
|
|
829
1708
|
}
|
|
@@ -833,18 +1712,44 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
833
1712
|
const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
|
|
834
1713
|
return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
|
|
835
1714
|
}
|
|
836
|
-
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true) {
|
|
837
|
-
|
|
1715
|
+
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
|
|
1716
|
+
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
1717
|
+
if (costOf(totalRows) <= budget) {
|
|
838
1718
|
return totalRows;
|
|
839
1719
|
}
|
|
840
|
-
if (
|
|
1720
|
+
if (costOf(0) > budget) {
|
|
841
1721
|
return 0;
|
|
842
1722
|
}
|
|
843
1723
|
let low = 0;
|
|
844
1724
|
let high = totalRows;
|
|
845
1725
|
while (high - low > 1) {
|
|
846
1726
|
const mid = Math.floor((low + high) / 2);
|
|
847
|
-
if (
|
|
1727
|
+
if (costOf(mid) <= budget) {
|
|
1728
|
+
low = mid;
|
|
1729
|
+
} else {
|
|
1730
|
+
high = mid;
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
return low;
|
|
1734
|
+
}
|
|
1735
|
+
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
|
|
1736
|
+
const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
|
|
1737
|
+
const fits = /* @__PURE__ */ __name((chunk) => {
|
|
1738
|
+
const live = Math.min(chunk * liveRowsPerChunk, covered);
|
|
1739
|
+
const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
|
|
1740
|
+
return cost <= budget;
|
|
1741
|
+
}, "fits");
|
|
1742
|
+
if (fits(covered)) {
|
|
1743
|
+
return covered;
|
|
1744
|
+
}
|
|
1745
|
+
if (!fits(1)) {
|
|
1746
|
+
return 0;
|
|
1747
|
+
}
|
|
1748
|
+
let low = 1;
|
|
1749
|
+
let high = covered;
|
|
1750
|
+
while (high - low > 1) {
|
|
1751
|
+
const mid = Math.floor((low + high) / 2);
|
|
1752
|
+
if (fits(mid)) {
|
|
848
1753
|
low = mid;
|
|
849
1754
|
} else {
|
|
850
1755
|
high = mid;
|
|
@@ -852,7 +1757,7 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
|
|
|
852
1757
|
}
|
|
853
1758
|
return low;
|
|
854
1759
|
}
|
|
855
|
-
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true) {
|
|
1760
|
+
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
|
|
856
1761
|
if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
|
|
857
1762
|
throw new exports.QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
|
|
858
1763
|
}
|
|
@@ -861,12 +1766,16 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
861
1766
|
}
|
|
862
1767
|
const budget = getMemoryBudget();
|
|
863
1768
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
864
|
-
const
|
|
865
|
-
const
|
|
1769
|
+
const rowsLive = live === null ? null : live.rows;
|
|
1770
|
+
const liveRowsPerChunk = live === null ? 1 : live.perChunk;
|
|
1771
|
+
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
1772
|
+
const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
|
|
1773
|
+
const externalMemory = estimateExternalMemory(liveRows, columnCount);
|
|
866
1774
|
const bounded = budget.candidates.map((candidate) => {
|
|
867
1775
|
const heapOnly = candidate.source === "V8 heap limit";
|
|
868
1776
|
return {
|
|
869
1777
|
...candidate,
|
|
1778
|
+
heapOnly,
|
|
870
1779
|
needs: heapOnly ? heapMemory : heapMemory + externalMemory,
|
|
871
1780
|
allowed: candidate.bytes * safetyFactor,
|
|
872
1781
|
bounds: heapOnly ? "the V8 heap" : "the whole process"
|
|
@@ -882,12 +1791,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
882
1791
|
const estimatedMemory = binding ? binding.needs : heapMemory;
|
|
883
1792
|
const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
|
|
884
1793
|
if (binding) {
|
|
1794
|
+
const includeExternal = !binding.heapOnly;
|
|
885
1795
|
const recommendedMaxRows = recommendedRowsFor(
|
|
886
1796
|
maxAllowedMemory,
|
|
887
1797
|
symbolTableSize,
|
|
888
1798
|
totalRows,
|
|
889
1799
|
columnCount,
|
|
890
|
-
materialisesRows
|
|
1800
|
+
materialisesRows,
|
|
1801
|
+
includeExternal
|
|
891
1802
|
);
|
|
892
1803
|
const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
|
|
893
1804
|
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
@@ -899,15 +1810,27 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
899
1810
|
const limitingScope = binding.bounds;
|
|
900
1811
|
const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
|
|
901
1812
|
const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
|
|
902
|
-
const nothingFits = recommendedMaxRows === 0;
|
|
903
1813
|
const containerBound = binding.source === "container memory limit";
|
|
1814
|
+
const chunked = rowsLive !== null;
|
|
1815
|
+
const recommendedChunk = chunked ? recommendedChunkFor(
|
|
1816
|
+
maxAllowedMemory,
|
|
1817
|
+
symbolTableSize,
|
|
1818
|
+
maxRows,
|
|
1819
|
+
totalRows,
|
|
1820
|
+
columnCount,
|
|
1821
|
+
liveRowsPerChunk,
|
|
1822
|
+
includeExternal
|
|
1823
|
+
) : 0;
|
|
1824
|
+
const knob = chunked ? "chunkSize" : "limit";
|
|
1825
|
+
const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
|
|
1826
|
+
const nothingFits = recommendedValue === 0;
|
|
904
1827
|
let advice;
|
|
905
1828
|
if (nothingFits) {
|
|
906
|
-
advice = `No row count fits this budget - the symbol table alone exceeds it, so
|
|
1829
|
+
advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
|
|
907
1830
|
} else if (containerBound) {
|
|
908
|
-
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or
|
|
1831
|
+
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
|
|
909
1832
|
} else {
|
|
910
|
-
advice = `Try
|
|
1833
|
+
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
911
1834
|
}
|
|
912
1835
|
throw new exports.QvdValidationError(
|
|
913
1836
|
`Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
@@ -927,32 +1850,55 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
927
1850
|
columnCount,
|
|
928
1851
|
totalRows,
|
|
929
1852
|
maxRows,
|
|
930
|
-
recommendedMaxRows
|
|
1853
|
+
recommendedMaxRows,
|
|
1854
|
+
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
1855
|
+
// recommendation for the other.
|
|
1856
|
+
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
931
1857
|
}
|
|
932
1858
|
);
|
|
933
1859
|
}
|
|
934
1860
|
}
|
|
1861
|
+
function formatCount(value) {
|
|
1862
|
+
return value.toLocaleString("en-US");
|
|
1863
|
+
}
|
|
935
1864
|
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
936
1865
|
const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
|
|
937
|
-
if (symbolTableSize
|
|
938
|
-
|
|
939
|
-
const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
|
|
940
|
-
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
941
|
-
const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
|
|
942
|
-
console.warn(
|
|
943
|
-
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). Loading all ${totalRows.toLocaleString()} rows will use ~${estimatedMB}MB RAM. Consider using the maxRows parameter for better performance and lower memory usage.`
|
|
944
|
-
);
|
|
1866
|
+
if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
1867
|
+
return;
|
|
945
1868
|
}
|
|
1869
|
+
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
1870
|
+
const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
|
|
1871
|
+
if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
1872
|
+
return;
|
|
1873
|
+
}
|
|
1874
|
+
const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
|
|
1875
|
+
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
1876
|
+
const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
|
|
1877
|
+
console.warn(
|
|
1878
|
+
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
1879
|
+
);
|
|
946
1880
|
}
|
|
947
1881
|
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
948
1882
|
var init_memoryUtils = __esm({
|
|
949
1883
|
"src/util/memoryUtils.js"() {
|
|
950
1884
|
init_QvdErrors();
|
|
1885
|
+
__name(getHeapLimit, "getHeapLimit");
|
|
1886
|
+
__name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
|
|
1887
|
+
__name(getMemoryBudget, "getMemoryBudget");
|
|
951
1888
|
HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
|
|
952
1889
|
MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
1890
|
+
__name(usableOldSpaceLimit, "usableOldSpaceLimit");
|
|
953
1891
|
BASE_BYTES = 16 * 1024 * 1024;
|
|
954
1892
|
ROW_BASE_BYTES = 72;
|
|
955
1893
|
PER_CELL_BYTES = 8;
|
|
1894
|
+
__name(estimateExternalMemory, "estimateExternalMemory");
|
|
1895
|
+
__name(estimateRowMemory, "estimateRowMemory");
|
|
1896
|
+
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
1897
|
+
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
1898
|
+
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
1899
|
+
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
1900
|
+
__name(formatCount, "formatCount");
|
|
1901
|
+
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
956
1902
|
}
|
|
957
1903
|
});
|
|
958
1904
|
|
|
@@ -983,7 +1929,7 @@ function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
|
|
|
983
1929
|
const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
|
|
984
1930
|
const heapMB = Math.round(heapLimit / 1024 / 1024);
|
|
985
1931
|
throw new exports.QvdValidationError(
|
|
986
|
-
`Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
|
|
1932
|
+
`Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
|
|
987
1933
|
{
|
|
988
1934
|
file: filePath,
|
|
989
1935
|
symbolTableSize: symbolTableLength,
|
|
@@ -1057,7 +2003,7 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
|
1057
2003
|
});
|
|
1058
2004
|
}
|
|
1059
2005
|
}
|
|
1060
|
-
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
|
|
2006
|
+
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
|
|
1061
2007
|
if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
|
|
1062
2008
|
throw new exports.QvdCorruptedError("Invalid record byte size", {
|
|
1063
2009
|
recordSize,
|
|
@@ -1119,23 +2065,28 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
|
|
|
1119
2065
|
}
|
|
1120
2066
|
}
|
|
1121
2067
|
const requiredIndexBytes = rowsToLoad * recordSize;
|
|
1122
|
-
|
|
2068
|
+
const bufferRecordStart = (windowFirstRow - bufferFirstRow) * recordSize;
|
|
2069
|
+
if (indexTableOffset + bufferRecordStart + requiredIndexBytes > bufferLength) {
|
|
1123
2070
|
throw new exports.QvdCorruptedError("Index table truncated", {
|
|
1124
2071
|
indexTableOffset,
|
|
1125
2072
|
requiredBytes: requiredIndexBytes,
|
|
1126
|
-
availableBytes: Math.max(0, bufferLength - indexTableOffset),
|
|
2073
|
+
availableBytes: Math.max(0, bufferLength - indexTableOffset - bufferRecordStart),
|
|
1127
2074
|
rowsToLoad,
|
|
2075
|
+
windowFirstRow,
|
|
2076
|
+
bufferFirstRow,
|
|
1128
2077
|
recordSize,
|
|
1129
2078
|
bufferSize: bufferLength,
|
|
1130
2079
|
file: filePath,
|
|
1131
2080
|
stage: "parseIndexTable"
|
|
1132
2081
|
});
|
|
1133
2082
|
}
|
|
1134
|
-
|
|
2083
|
+
const requiredTableBytes = (windowFirstRow + rowsToLoad) * recordSize;
|
|
2084
|
+
if (indexTableLength < requiredTableBytes) {
|
|
1135
2085
|
throw new exports.QvdCorruptedError("Index table length smaller than required", {
|
|
1136
2086
|
indexTableLength,
|
|
1137
|
-
requiredBytes:
|
|
2087
|
+
requiredBytes: requiredTableBytes,
|
|
1138
2088
|
rowsToLoad,
|
|
2089
|
+
windowFirstRow,
|
|
1139
2090
|
recordSize,
|
|
1140
2091
|
file: filePath,
|
|
1141
2092
|
stage: "parseIndexTable"
|
|
@@ -1196,262 +2147,235 @@ var init_validationUtils = __esm({
|
|
|
1196
2147
|
init_QvdErrors();
|
|
1197
2148
|
init_memoryUtils();
|
|
1198
2149
|
init_bitUtils();
|
|
2150
|
+
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2151
|
+
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2152
|
+
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2153
|
+
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2154
|
+
__name(validateRecordCount, "validateRecordCount");
|
|
2155
|
+
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2156
|
+
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
1199
2157
|
}
|
|
1200
2158
|
});
|
|
1201
2159
|
|
|
1202
2160
|
// src/util/symbolParser.js
|
|
1203
|
-
function
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
bufferSize: bufferLength,
|
|
1209
|
-
file: filePath,
|
|
1210
|
-
stage: "parseSymbolTable"
|
|
1211
|
-
});
|
|
1212
|
-
}
|
|
1213
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1214
|
-
const value = Buffer.from(byteData).readIntLE(0, byteData.length);
|
|
1215
|
-
return { symbol: exports.QvdSymbol.fromIntValue(value), bytesRead: 4 };
|
|
1216
|
-
}
|
|
1217
|
-
function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1218
|
-
if (pointer + 8 > bufferLength) {
|
|
1219
|
-
throw new exports.QvdCorruptedError("Buffer overflow reading double symbol", {
|
|
2161
|
+
function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
|
|
2162
|
+
const bufferLength = symbolBuffer.length;
|
|
2163
|
+
const found = symbolBuffer.indexOf(0, from);
|
|
2164
|
+
if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
|
|
2165
|
+
throw new exports.QvdCorruptedError(`${kind} exceeds maximum length`, {
|
|
1220
2166
|
field: fieldName,
|
|
1221
|
-
|
|
1222
|
-
bufferSize: bufferLength,
|
|
1223
|
-
file: filePath,
|
|
1224
|
-
stage: "parseSymbolTable"
|
|
1225
|
-
});
|
|
1226
|
-
}
|
|
1227
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1228
|
-
const value = Buffer.from(byteData).readDoubleLE(0);
|
|
1229
|
-
return { symbol: exports.QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
|
|
1230
|
-
}
|
|
1231
|
-
function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1232
|
-
const startPointer = pointer;
|
|
1233
|
-
const maxStringLength = 1048576;
|
|
1234
|
-
let stringLength = 0;
|
|
1235
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1236
|
-
if (stringLength >= maxStringLength) {
|
|
1237
|
-
throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1238
|
-
field: fieldName,
|
|
1239
|
-
maxLength: maxStringLength,
|
|
1240
|
-
file: filePath,
|
|
1241
|
-
stage: "parseSymbolTable"
|
|
1242
|
-
});
|
|
1243
|
-
}
|
|
1244
|
-
pointer++;
|
|
1245
|
-
stringLength++;
|
|
1246
|
-
}
|
|
1247
|
-
if (pointer >= bufferLength) {
|
|
1248
|
-
throw new exports.QvdCorruptedError("String symbol not null-terminated", {
|
|
1249
|
-
field: fieldName,
|
|
1250
|
-
pointer,
|
|
1251
|
-
bufferSize: bufferLength,
|
|
2167
|
+
maxLength: MAX_TEXT_BYTES,
|
|
1252
2168
|
file: filePath,
|
|
1253
2169
|
stage: "parseSymbolTable"
|
|
1254
2170
|
});
|
|
1255
2171
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
}
|
|
1259
|
-
function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1260
|
-
const startPointer = pointer;
|
|
1261
|
-
const maxStringLength = 1048576;
|
|
1262
|
-
let stringLength = 0;
|
|
1263
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1264
|
-
if (stringLength >= maxStringLength) {
|
|
1265
|
-
throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1266
|
-
field: fieldName,
|
|
1267
|
-
maxLength: maxStringLength,
|
|
1268
|
-
file: filePath,
|
|
1269
|
-
stage: "parseSymbolTable"
|
|
1270
|
-
});
|
|
1271
|
-
}
|
|
1272
|
-
pointer++;
|
|
1273
|
-
stringLength++;
|
|
1274
|
-
}
|
|
1275
|
-
if (pointer >= bufferLength) {
|
|
1276
|
-
throw new exports.QvdCorruptedError("String symbol not null-terminated", {
|
|
2172
|
+
if (found === -1) {
|
|
2173
|
+
throw new exports.QvdCorruptedError(`${kind} not null-terminated`, {
|
|
1277
2174
|
field: fieldName,
|
|
1278
|
-
pointer,
|
|
2175
|
+
pointer: bufferLength,
|
|
1279
2176
|
bufferSize: bufferLength,
|
|
1280
2177
|
file: filePath,
|
|
1281
2178
|
stage: "parseSymbolTable"
|
|
1282
2179
|
});
|
|
1283
2180
|
}
|
|
1284
|
-
return
|
|
2181
|
+
return found;
|
|
1285
2182
|
}
|
|
1286
|
-
function
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
});
|
|
1295
|
-
}
|
|
1296
|
-
const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1297
|
-
const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
|
|
1298
|
-
pointer += 4;
|
|
1299
|
-
const stringStart = pointer;
|
|
1300
|
-
const maxStringLength = 1048576;
|
|
1301
|
-
let stringLength = 0;
|
|
1302
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1303
|
-
if (stringLength >= maxStringLength) {
|
|
1304
|
-
throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1305
|
-
field: fieldName,
|
|
1306
|
-
maxLength: maxStringLength,
|
|
1307
|
-
file: filePath,
|
|
1308
|
-
stage: "parseSymbolTable"
|
|
1309
|
-
});
|
|
1310
|
-
}
|
|
1311
|
-
pointer++;
|
|
1312
|
-
stringLength++;
|
|
1313
|
-
}
|
|
1314
|
-
if (pointer >= bufferLength) {
|
|
1315
|
-
throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1316
|
-
field: fieldName,
|
|
1317
|
-
pointer,
|
|
1318
|
-
bufferSize: bufferLength,
|
|
1319
|
-
file: filePath,
|
|
1320
|
-
stage: "parseSymbolTable"
|
|
1321
|
-
});
|
|
1322
|
-
}
|
|
1323
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1324
|
-
return { symbol: exports.QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
|
|
2183
|
+
function overflow(message, pointer, bufferLength, fieldName, filePath) {
|
|
2184
|
+
throw new exports.QvdCorruptedError(message, {
|
|
2185
|
+
field: fieldName,
|
|
2186
|
+
pointer,
|
|
2187
|
+
bufferSize: bufferLength,
|
|
2188
|
+
file: filePath,
|
|
2189
|
+
stage: "parseSymbolTable"
|
|
2190
|
+
});
|
|
1325
2191
|
}
|
|
1326
|
-
function
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
2192
|
+
function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
|
|
2193
|
+
const bufferLength = symbolBuffer.length;
|
|
2194
|
+
const numbers = [];
|
|
2195
|
+
const texts = [];
|
|
2196
|
+
let pointer = start;
|
|
2197
|
+
while (pointer < end) {
|
|
2198
|
+
const typeByte = symbolBuffer[pointer++];
|
|
2199
|
+
const decode = keep === null || keep.has(numbers.length);
|
|
2200
|
+
let number = null;
|
|
2201
|
+
let text = null;
|
|
2202
|
+
switch (typeByte) {
|
|
2203
|
+
case 1: {
|
|
2204
|
+
if (decode) {
|
|
2205
|
+
if (pointer + 4 > bufferLength) {
|
|
2206
|
+
overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
|
|
2207
|
+
}
|
|
2208
|
+
number = symbolBuffer.readInt32LE(pointer);
|
|
2209
|
+
}
|
|
2210
|
+
pointer += 4;
|
|
2211
|
+
break;
|
|
2212
|
+
}
|
|
2213
|
+
case 2: {
|
|
2214
|
+
if (decode) {
|
|
2215
|
+
if (pointer + 8 > bufferLength) {
|
|
2216
|
+
overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
|
|
2217
|
+
}
|
|
2218
|
+
number = symbolBuffer.readDoubleLE(pointer);
|
|
2219
|
+
}
|
|
2220
|
+
pointer += 8;
|
|
2221
|
+
break;
|
|
2222
|
+
}
|
|
2223
|
+
case 4: {
|
|
2224
|
+
const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
|
|
2225
|
+
if (decode) {
|
|
2226
|
+
text = symbolBuffer.toString("utf8", pointer, terminator);
|
|
2227
|
+
}
|
|
2228
|
+
pointer = terminator + 1;
|
|
2229
|
+
break;
|
|
2230
|
+
}
|
|
2231
|
+
case 5:
|
|
2232
|
+
case 6: {
|
|
2233
|
+
const numberBytes = typeByte === 5 ? 4 : 8;
|
|
2234
|
+
if (pointer + numberBytes > bufferLength) {
|
|
2235
|
+
const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
|
|
2236
|
+
overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
|
|
2237
|
+
}
|
|
2238
|
+
const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
|
|
2239
|
+
if (decode) {
|
|
2240
|
+
number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
|
|
2241
|
+
text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
|
|
2242
|
+
}
|
|
2243
|
+
pointer = terminator + 1;
|
|
2244
|
+
break;
|
|
2245
|
+
}
|
|
2246
|
+
default: {
|
|
2247
|
+
throw new exports.QvdParseError("Unknown symbol type byte", {
|
|
2248
|
+
typeByte: typeByte.toString(16),
|
|
2249
|
+
offset: pointer - 1,
|
|
2250
|
+
file: filePath,
|
|
2251
|
+
stage: "parseSymbolTable"
|
|
2252
|
+
});
|
|
2253
|
+
}
|
|
1350
2254
|
}
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
}
|
|
1354
|
-
if (pointer >= bufferLength) {
|
|
1355
|
-
throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1356
|
-
field: fieldName,
|
|
1357
|
-
pointer,
|
|
1358
|
-
bufferSize: bufferLength,
|
|
1359
|
-
file: filePath,
|
|
1360
|
-
stage: "parseSymbolTable"
|
|
1361
|
-
});
|
|
2255
|
+
numbers.push(number);
|
|
2256
|
+
texts.push(text);
|
|
1362
2257
|
}
|
|
1363
|
-
|
|
1364
|
-
return {
|
|
1365
|
-
symbol: exports.QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
|
|
1366
|
-
bytesRead: pointer - (stringStart - 8) + 1
|
|
1367
|
-
};
|
|
2258
|
+
return { numbers, texts };
|
|
1368
2259
|
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
});
|
|
2260
|
+
var MAX_TEXT_BYTES;
|
|
2261
|
+
var init_symbolParser = __esm({
|
|
2262
|
+
"src/util/symbolParser.js"() {
|
|
2263
|
+
init_QvdErrors();
|
|
2264
|
+
MAX_TEXT_BYTES = 1048576;
|
|
2265
|
+
__name(textEnd, "textEnd");
|
|
2266
|
+
__name(overflow, "overflow");
|
|
2267
|
+
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
1378
2268
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
2269
|
+
});
|
|
2270
|
+
|
|
2271
|
+
// src/util/resolveSymbols.js
|
|
2272
|
+
function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
|
|
2273
|
+
const length = symbols.numbers.length;
|
|
2274
|
+
const values = new Array(length);
|
|
2275
|
+
const entryValues = [];
|
|
2276
|
+
const numbers = [];
|
|
2277
|
+
const texts = [];
|
|
2278
|
+
let pure = 0;
|
|
2279
|
+
let partial = false;
|
|
2280
|
+
for (let index = 0; index < length; index++) {
|
|
2281
|
+
const text = symbols.texts[index];
|
|
2282
|
+
const number = symbols.numbers[index];
|
|
2283
|
+
if (text === null) {
|
|
2284
|
+
values[index] = number === null ? void 0 : number;
|
|
2285
|
+
if (number !== null) pure++;
|
|
2286
|
+
continue;
|
|
2287
|
+
}
|
|
2288
|
+
if (number === null) {
|
|
2289
|
+
if (coerce && isNumericText(text)) {
|
|
2290
|
+
values[index] = Number(text);
|
|
2291
|
+
entryValues.push(values[index]);
|
|
2292
|
+
numbers.push(null);
|
|
2293
|
+
texts.push(text);
|
|
2294
|
+
partial = true;
|
|
2295
|
+
} else {
|
|
2296
|
+
values[index] = text;
|
|
2297
|
+
pure++;
|
|
2298
|
+
}
|
|
2299
|
+
continue;
|
|
2300
|
+
}
|
|
2301
|
+
partial = true;
|
|
2302
|
+
if (mode === "both") {
|
|
2303
|
+
values[index] = dualFromSymbol(number, text);
|
|
2304
|
+
continue;
|
|
1391
2305
|
}
|
|
1392
|
-
|
|
1393
|
-
|
|
2306
|
+
values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
|
|
2307
|
+
entryValues.push(values[index]);
|
|
2308
|
+
numbers.push(number);
|
|
2309
|
+
texts.push(text);
|
|
1394
2310
|
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
2311
|
+
let entry = null;
|
|
2312
|
+
if (entryValues.length > 0) {
|
|
2313
|
+
entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
|
|
2314
|
+
field,
|
|
2315
|
+
values: Object.freeze(entryValues),
|
|
2316
|
+
numbers: Object.freeze(numbers),
|
|
2317
|
+
texts: Object.freeze(texts)
|
|
1402
2318
|
});
|
|
1403
2319
|
}
|
|
1404
|
-
return
|
|
2320
|
+
return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
|
|
1405
2321
|
}
|
|
1406
|
-
function
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
return parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1411
|
-
}
|
|
1412
|
-
return { symbol: null, bytesRead: 4 };
|
|
1413
|
-
}
|
|
1414
|
-
case 2: {
|
|
1415
|
-
if (shouldParse) {
|
|
1416
|
-
return parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1417
|
-
}
|
|
1418
|
-
return { symbol: null, bytesRead: 8 };
|
|
2322
|
+
function collides(symbols, values, recorded) {
|
|
2323
|
+
for (let index = 0; index < values.length; index++) {
|
|
2324
|
+
if (!isPure(symbols, index, values[index])) {
|
|
2325
|
+
continue;
|
|
1419
2326
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
return parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1423
|
-
}
|
|
1424
|
-
const bytesRead = skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1425
|
-
return { symbol: null, bytesRead };
|
|
2327
|
+
if (recorded.has(values[index])) {
|
|
2328
|
+
return true;
|
|
1426
2329
|
}
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
2330
|
+
}
|
|
2331
|
+
return false;
|
|
2332
|
+
}
|
|
2333
|
+
function isPure(symbols, index, value) {
|
|
2334
|
+
const text = symbols.texts[index];
|
|
2335
|
+
const number = symbols.numbers[index];
|
|
2336
|
+
return text === null ? number !== null : number === null && value === text;
|
|
2337
|
+
}
|
|
2338
|
+
function collisionEntry(symbols, field, values, mode) {
|
|
2339
|
+
const recorded = /* @__PURE__ */ new Set();
|
|
2340
|
+
const inPassA = values.map((value, index) => {
|
|
2341
|
+
if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
|
|
2342
|
+
return false;
|
|
1433
2343
|
}
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
}
|
|
1438
|
-
const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
|
|
1439
|
-
return { symbol: null, bytesRead };
|
|
2344
|
+
const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
|
|
2345
|
+
if (recordedHere) {
|
|
2346
|
+
recorded.add(values[index]);
|
|
1440
2347
|
}
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
2348
|
+
return recordedHere;
|
|
2349
|
+
});
|
|
2350
|
+
const entryValues = [];
|
|
2351
|
+
const numbers = [];
|
|
2352
|
+
const texts = [];
|
|
2353
|
+
for (let index = 0; index < values.length; index++) {
|
|
2354
|
+
if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
|
|
2355
|
+
entryValues.push(values[index]);
|
|
2356
|
+
numbers.push(symbols.numbers[index]);
|
|
2357
|
+
texts.push(symbols.texts[index]);
|
|
1448
2358
|
}
|
|
1449
2359
|
}
|
|
2360
|
+
return Object.freeze({
|
|
2361
|
+
field,
|
|
2362
|
+
values: Object.freeze(entryValues),
|
|
2363
|
+
numbers: Object.freeze(numbers),
|
|
2364
|
+
texts: Object.freeze(texts)
|
|
2365
|
+
});
|
|
1450
2366
|
}
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
2367
|
+
function symbolHalves(symbols) {
|
|
2368
|
+
return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
|
|
2369
|
+
}
|
|
2370
|
+
var init_resolveSymbols = __esm({
|
|
2371
|
+
"src/util/resolveSymbols.js"() {
|
|
2372
|
+
init_QvdDual();
|
|
2373
|
+
init_cellRules();
|
|
2374
|
+
__name(resolveFieldSymbols, "resolveFieldSymbols");
|
|
2375
|
+
__name(collides, "collides");
|
|
2376
|
+
__name(isPure, "isPure");
|
|
2377
|
+
__name(collisionEntry, "collisionEntry");
|
|
2378
|
+
__name(symbolHalves, "symbolHalves");
|
|
1455
2379
|
}
|
|
1456
2380
|
});
|
|
1457
2381
|
|
|
@@ -1461,20 +2385,49 @@ __export(QvdColumnTable_exports, {
|
|
|
1461
2385
|
QvdColumn: () => exports.QvdColumn,
|
|
1462
2386
|
QvdColumnTable: () => exports.QvdColumnTable
|
|
1463
2387
|
});
|
|
2388
|
+
function numberOf(value, number) {
|
|
2389
|
+
if (typeof value === "number") {
|
|
2390
|
+
return value;
|
|
2391
|
+
}
|
|
2392
|
+
if (typeof value === "string") {
|
|
2393
|
+
return number ?? NaN;
|
|
2394
|
+
}
|
|
2395
|
+
const dual = asDual(value);
|
|
2396
|
+
return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
|
|
2397
|
+
}
|
|
2398
|
+
function textOf(value) {
|
|
2399
|
+
if (typeof value === "string") {
|
|
2400
|
+
return value;
|
|
2401
|
+
}
|
|
2402
|
+
const dual = asDual(value);
|
|
2403
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
2404
|
+
}
|
|
1464
2405
|
exports.QvdColumn = void 0; exports.QvdColumnTable = void 0;
|
|
1465
2406
|
var init_QvdColumnTable = __esm({
|
|
1466
2407
|
"src/QvdColumnTable.js"() {
|
|
1467
2408
|
init_QvdErrors();
|
|
2409
|
+
init_cellRules();
|
|
2410
|
+
init_readOptions();
|
|
2411
|
+
__name(numberOf, "numberOf");
|
|
2412
|
+
__name(textOf, "textOf");
|
|
1468
2413
|
exports.QvdColumn = class {
|
|
2414
|
+
static {
|
|
2415
|
+
__name(this, "QvdColumn");
|
|
2416
|
+
}
|
|
1469
2417
|
/**
|
|
1470
2418
|
* @param {string} name The field name.
|
|
1471
2419
|
* @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
|
|
1472
2420
|
* @param {Array<any>} symbols The field's distinct values, indexed by code.
|
|
2421
|
+
* @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
|
|
2422
|
+
* field whose values do not show them all - a dual read as one half, a string read as a number.
|
|
2423
|
+
* Without them, the halves are derived from the values: a string is its own text, a dual has its
|
|
2424
|
+
* own, and a number has none.
|
|
1473
2425
|
*/
|
|
1474
|
-
constructor(name, codes, symbols) {
|
|
2426
|
+
constructor(name, codes, symbols, halves = null) {
|
|
1475
2427
|
this._name = name;
|
|
1476
2428
|
this._codes = codes;
|
|
1477
2429
|
this._symbols = symbols;
|
|
2430
|
+
this._halves = halves;
|
|
1478
2431
|
Object.freeze(this);
|
|
1479
2432
|
}
|
|
1480
2433
|
/** @return {string} The field name. */
|
|
@@ -1501,6 +2454,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1501
2454
|
*
|
|
1502
2455
|
* One entry per distinct value, not per row: a few thousand entries for a column of millions.
|
|
1503
2456
|
*
|
|
2457
|
+
* A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
|
|
2458
|
+
* entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
|
|
2459
|
+
*
|
|
1504
2460
|
* @return {ReadonlyArray<any>} The dictionary.
|
|
1505
2461
|
*/
|
|
1506
2462
|
get symbols() {
|
|
@@ -1524,6 +2480,42 @@ var init_QvdColumnTable = __esm({
|
|
|
1524
2480
|
const code = this._codes[row];
|
|
1525
2481
|
return code < 0 ? null : this._symbols[code];
|
|
1526
2482
|
}
|
|
2483
|
+
/**
|
|
2484
|
+
* The text of one row: the text Qlik displays for its value.
|
|
2485
|
+
*
|
|
2486
|
+
* A string is its own text and a dual has its own. A value read as one half of a symbol - a date
|
|
2487
|
+
* read as its serial, a string read as a number - has the text the file stores for it, and a pure
|
|
2488
|
+
* number has none.
|
|
2489
|
+
*
|
|
2490
|
+
* @param {number} row The row index.
|
|
2491
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
2492
|
+
* @throws {QvdValidationError} If the row is not an integer within the column.
|
|
2493
|
+
*/
|
|
2494
|
+
textAt(row) {
|
|
2495
|
+
if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
|
|
2496
|
+
throw new exports.QvdValidationError("Row index out of bounds", {
|
|
2497
|
+
column: this._name,
|
|
2498
|
+
row,
|
|
2499
|
+
length: this._codes.length
|
|
2500
|
+
});
|
|
2501
|
+
}
|
|
2502
|
+
const code = this._codes[row];
|
|
2503
|
+
if (code < 0) {
|
|
2504
|
+
return null;
|
|
2505
|
+
}
|
|
2506
|
+
return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
|
|
2507
|
+
}
|
|
2508
|
+
/**
|
|
2509
|
+
* The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
|
|
2510
|
+
*
|
|
2511
|
+
* @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
|
|
2512
|
+
*/
|
|
2513
|
+
symbolTexts() {
|
|
2514
|
+
if (this._halves !== null) {
|
|
2515
|
+
return this._halves.texts;
|
|
2516
|
+
}
|
|
2517
|
+
return Object.freeze(this._symbols.map(textOf));
|
|
2518
|
+
}
|
|
1527
2519
|
/**
|
|
1528
2520
|
* Iterates the column's values without materialising it.
|
|
1529
2521
|
*
|
|
@@ -1567,6 +2559,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1567
2559
|
* one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
|
|
1568
2560
|
* codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
|
|
1569
2561
|
*
|
|
2562
|
+
* A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
|
|
2563
|
+
* `{duals: 'text'}` gives the serial the file stores for it, not NaN.
|
|
2564
|
+
*
|
|
1570
2565
|
* Scanning `codes` against this is the fastest way to read a column, because both sides are
|
|
1571
2566
|
* contiguous typed arrays and the dictionary fits in cache:
|
|
1572
2567
|
*
|
|
@@ -1588,8 +2583,7 @@ var init_QvdColumnTable = __esm({
|
|
|
1588
2583
|
numericSymbols() {
|
|
1589
2584
|
const out = new Float64Array(this._symbols.length);
|
|
1590
2585
|
for (let index = 0; index < this._symbols.length; index++) {
|
|
1591
|
-
|
|
1592
|
-
out[index] = typeof value === "number" ? value : NaN;
|
|
2586
|
+
out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
|
|
1593
2587
|
}
|
|
1594
2588
|
return out;
|
|
1595
2589
|
}
|
|
@@ -1604,7 +2598,8 @@ var init_QvdColumnTable = __esm({
|
|
|
1604
2598
|
* @param {Object} [options] Conversion options.
|
|
1605
2599
|
* @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
|
|
1606
2600
|
* number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
|
|
1607
|
-
* NaN, which is the right choice only when the caller knows the column is numeric.
|
|
2601
|
+
* NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
|
|
2602
|
+
* number here, as it is to `numericSymbols`.
|
|
1608
2603
|
* @return {Float64Array} One number per row.
|
|
1609
2604
|
* @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
|
|
1610
2605
|
*/
|
|
@@ -1624,6 +2619,11 @@ var init_QvdColumnTable = __esm({
|
|
|
1624
2619
|
out[row] = value;
|
|
1625
2620
|
continue;
|
|
1626
2621
|
}
|
|
2622
|
+
const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
|
|
2623
|
+
if (!Number.isNaN(number)) {
|
|
2624
|
+
out[row] = number;
|
|
2625
|
+
continue;
|
|
2626
|
+
}
|
|
1627
2627
|
if (onNonNumeric === "throw") {
|
|
1628
2628
|
throw new exports.QvdValidationError("Column holds a value that is not a number", {
|
|
1629
2629
|
column: this._name,
|
|
@@ -1639,29 +2639,57 @@ var init_QvdColumnTable = __esm({
|
|
|
1639
2639
|
}
|
|
1640
2640
|
};
|
|
1641
2641
|
exports.QvdColumnTable = class {
|
|
2642
|
+
static {
|
|
2643
|
+
__name(this, "QvdColumnTable");
|
|
2644
|
+
}
|
|
1642
2645
|
/**
|
|
1643
2646
|
* @param {Object} decoded What the reader decoded.
|
|
1644
2647
|
* @param {Array<string>} decoded.columns Field names, in file order.
|
|
1645
2648
|
* @param {Array<Int32Array>} decoded.codesByField One code array per field.
|
|
1646
2649
|
* @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
|
|
2650
|
+
* @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
|
|
2651
|
+
* or null for a field whose values show them.
|
|
1647
2652
|
* @param {number} decoded.rowCount Rows decoded.
|
|
1648
2653
|
* @param {any} decoded.metadata The raw QvdTableHeader.
|
|
2654
|
+
* @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
|
|
2655
|
+
* stored-symbol record, as a data frame of the same read carries it.
|
|
1649
2656
|
* @param {any} decoded.loadStats Statistics about the read.
|
|
1650
2657
|
*/
|
|
1651
|
-
constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
|
|
2658
|
+
constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
|
|
1652
2659
|
this._columns = columns;
|
|
1653
2660
|
this._codesByField = codesByField;
|
|
1654
2661
|
this._symbolsByField = symbolsByField;
|
|
2662
|
+
this._halvesByField = halvesByField ?? null;
|
|
1655
2663
|
this._rowCount = rowCount;
|
|
1656
2664
|
this._metadata = metadata;
|
|
2665
|
+
this._storedSymbols = storedSymbols ?? null;
|
|
1657
2666
|
this._loadStats = loadStats;
|
|
1658
2667
|
}
|
|
1659
2668
|
/**
|
|
1660
2669
|
* Reads a QVD file as columns.
|
|
1661
2670
|
*
|
|
2671
|
+
* Takes the same options as `QvdDataFrame.fromQvd`, with the same meanings - one option
|
|
2672
|
+
* vocabulary for both read paths, because they are two answers about the same file rather than
|
|
2673
|
+
* two features. `{offset, limit}` is how a caller pages through a file columnwise; there is no
|
|
2674
|
+
* columnar `iterate()` because there is nothing for it to bound - a columnar read materialises
|
|
2675
|
+
* no rows, which is the memory chunking exists to cap.
|
|
2676
|
+
*
|
|
1662
2677
|
* @param {string} path The path to the QVD file.
|
|
1663
2678
|
* @param {Object} [options] Loading options, with the same meanings they have on `fromQvd`.
|
|
1664
|
-
* @param {number|null} [options.maxRows]
|
|
2679
|
+
* @param {number|null} [options.maxRows] Rows to decode. The older name for `limit`.
|
|
2680
|
+
* @param {number|null} [options.limit] Rows to decode, counting from `offset`.
|
|
2681
|
+
* @param {number} [options.offset] File row to start at.
|
|
2682
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
2683
|
+
* appear. Unselected fields have their symbols skipped entirely.
|
|
2684
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
|
|
2685
|
+
* number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
|
|
2686
|
+
* text and `numericSymbols` the number. Anything else throws.
|
|
2687
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
|
|
2688
|
+
* number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
2689
|
+
* `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
|
|
2690
|
+
* Anything but a boolean throws.
|
|
2691
|
+
* @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
|
|
2692
|
+
* @param {AbortSignal} [options.signal] Cancels the read.
|
|
1665
2693
|
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
1666
2694
|
* @param {number} [options.memorySafetyFactor] Fraction of the memory budget a load may use.
|
|
1667
2695
|
* @param {number} [options.symbolFilteringThreshold] Symbol table size above which a limited
|
|
@@ -1671,15 +2699,13 @@ var init_QvdColumnTable = __esm({
|
|
|
1671
2699
|
static async fromQvd(path3, options = {}) {
|
|
1672
2700
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
1673
2701
|
const reader = new QvdFileReader2(path3, {
|
|
1674
|
-
|
|
1675
|
-
memorySafetyFactor: options.memorySafetyFactor,
|
|
1676
|
-
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
2702
|
+
...readerOptionsFrom(options),
|
|
1677
2703
|
// This read builds no rows, so the memory guard must not charge it for them. A columnar
|
|
1678
2704
|
// read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
|
|
1679
2705
|
// refused below a 2GB one.
|
|
1680
2706
|
materialisesRows: false
|
|
1681
2707
|
});
|
|
1682
|
-
return await reader.loadColumnar(options
|
|
2708
|
+
return await reader.loadColumnar(windowFrom(options));
|
|
1683
2709
|
}
|
|
1684
2710
|
/** @return {Array<string>} Field names, in file order. */
|
|
1685
2711
|
get columns() {
|
|
@@ -1701,6 +2727,14 @@ var init_QvdColumnTable = __esm({
|
|
|
1701
2727
|
get loadStats() {
|
|
1702
2728
|
return this._loadStats;
|
|
1703
2729
|
}
|
|
2730
|
+
/**
|
|
2731
|
+
* The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
|
|
2732
|
+
*
|
|
2733
|
+
* @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
|
|
2734
|
+
*/
|
|
2735
|
+
get storedSymbols() {
|
|
2736
|
+
return this._storedSymbols;
|
|
2737
|
+
}
|
|
1704
2738
|
/**
|
|
1705
2739
|
* One column.
|
|
1706
2740
|
*
|
|
@@ -1716,7 +2750,12 @@ var init_QvdColumnTable = __esm({
|
|
|
1716
2750
|
availableColumns: this._columns
|
|
1717
2751
|
});
|
|
1718
2752
|
}
|
|
1719
|
-
return new exports.QvdColumn(
|
|
2753
|
+
return new exports.QvdColumn(
|
|
2754
|
+
name,
|
|
2755
|
+
this._codesByField[index],
|
|
2756
|
+
this._symbolsByField[index],
|
|
2757
|
+
this._halvesByField?.[index] ?? null
|
|
2758
|
+
);
|
|
1720
2759
|
}
|
|
1721
2760
|
};
|
|
1722
2761
|
}
|
|
@@ -1727,7 +2766,17 @@ var QvdFileReader_exports = {};
|
|
|
1727
2766
|
__export(QvdFileReader_exports, {
|
|
1728
2767
|
QvdFileReader: () => exports.QvdFileReader
|
|
1729
2768
|
});
|
|
1730
|
-
|
|
2769
|
+
function closeReadStream(stream) {
|
|
2770
|
+
if (stream.closed) {
|
|
2771
|
+
return Promise.resolve();
|
|
2772
|
+
}
|
|
2773
|
+
return new Promise((resolve) => {
|
|
2774
|
+
stream.once("close", () => resolve());
|
|
2775
|
+
stream.once("error", () => resolve());
|
|
2776
|
+
stream.destroy();
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS; exports.QvdFileReader = void 0;
|
|
1731
2780
|
var init_QvdFileReader = __esm({
|
|
1732
2781
|
"src/QvdFileReader.js"() {
|
|
1733
2782
|
init_QvdDataFrame();
|
|
@@ -1737,9 +2786,17 @@ var init_QvdFileReader = __esm({
|
|
|
1737
2786
|
init_memoryUtils();
|
|
1738
2787
|
init_validationUtils();
|
|
1739
2788
|
init_symbolParser();
|
|
2789
|
+
init_readOptions();
|
|
2790
|
+
init_resolveSymbols();
|
|
2791
|
+
init_storedSymbols();
|
|
1740
2792
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
1741
2793
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
2794
|
+
ANALYSIS_SLICE_ROWS = 65536;
|
|
2795
|
+
__name(closeReadStream, "closeReadStream");
|
|
1742
2796
|
exports.QvdFileReader = class {
|
|
2797
|
+
static {
|
|
2798
|
+
__name(this, "QvdFileReader");
|
|
2799
|
+
}
|
|
1743
2800
|
/**
|
|
1744
2801
|
* Constructs a new QVD file parser.
|
|
1745
2802
|
*
|
|
@@ -1750,9 +2807,10 @@ var init_QvdFileReader = __esm({
|
|
|
1750
2807
|
* points outside it is rejected. Defaults to the current working directory. To permit
|
|
1751
2808
|
* an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
|
|
1752
2809
|
* empty value falls back to the working directory rather than removing the restriction.
|
|
1753
|
-
* @param {number} [options.memorySafetyFactor=0.
|
|
1754
|
-
* load may use. The budget is the
|
|
1755
|
-
*
|
|
2810
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction (0.0-1.0) of the memory budget a
|
|
2811
|
+
* load may use. The budget is the smaller of the V8 heap limit and any container memory limit;
|
|
2812
|
+
* what the OS reports as available is recorded for diagnostics and deliberately not allowed to
|
|
2813
|
+
* bind - see `getMemoryBudget`. Default is 0.8. **Zero disables the memory
|
|
1756
2814
|
* check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
|
|
1757
2815
|
* Bun reports its current heap as its heap limit - and for callers who would rather manage
|
|
1758
2816
|
* memory themselves than trust the estimate.
|
|
@@ -1763,41 +2821,123 @@ var init_QvdFileReader = __esm({
|
|
|
1763
2821
|
* above which a lazy load switches to the two-pass filtering path. The default of 50MB is
|
|
1764
2822
|
* the point where the extra analysis pass pays for itself; lower it to use filtering on
|
|
1765
2823
|
* smaller files, raise it to keep the simpler single-pass read for longer.
|
|
2824
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
2825
|
+
* appear. Null reads every field, in file order. An unknown or repeated name is refused.
|
|
2826
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
2827
|
+
* text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
|
|
2828
|
+
* value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
|
|
2829
|
+
* `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
|
|
2830
|
+
* `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
|
|
2831
|
+
* the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
|
|
2832
|
+
* throws a `QvdValidationError`.
|
|
2833
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
2834
|
+
* reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
|
|
2835
|
+
* every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
|
|
2836
|
+
* dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
|
|
2837
|
+
* dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
|
|
2838
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
2839
|
+
* read proceeds - the same shape `QvdFileWriter` emits.
|
|
2840
|
+
* @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
|
|
2841
|
+
* `signal.reason`, exactly as `signal.throwIfAborted()` does.
|
|
1766
2842
|
*/
|
|
1767
2843
|
constructor(filePath, options = {}) {
|
|
1768
2844
|
const {
|
|
1769
2845
|
allowedDir,
|
|
1770
2846
|
memorySafetyFactor = 0.8,
|
|
1771
2847
|
symbolFilteringThreshold = 50 * 1024 * 1024,
|
|
1772
|
-
materialisesRows = true
|
|
2848
|
+
materialisesRows = true,
|
|
2849
|
+
fields = null,
|
|
2850
|
+
duals,
|
|
2851
|
+
coerceNumericStrings,
|
|
2852
|
+
onProgress,
|
|
2853
|
+
signal
|
|
1773
2854
|
} = options;
|
|
1774
2855
|
this._materialisesRows = materialisesRows;
|
|
1775
2856
|
this._path = validatePath(filePath, allowedDir);
|
|
2857
|
+
this._duals = normaliseDuals(duals, this._path);
|
|
2858
|
+
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
1776
2859
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
1777
2860
|
this._symbolFilteringThreshold = symbolFilteringThreshold;
|
|
2861
|
+
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
2862
|
+
throw new exports.QvdValidationError("onProgress must be a function", {
|
|
2863
|
+
provided: onProgress,
|
|
2864
|
+
type: typeof onProgress,
|
|
2865
|
+
file: this._path
|
|
2866
|
+
});
|
|
2867
|
+
}
|
|
2868
|
+
if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
|
|
2869
|
+
throw new exports.QvdValidationError("signal must be an AbortSignal", {
|
|
2870
|
+
provided: signal,
|
|
2871
|
+
type: typeof signal,
|
|
2872
|
+
file: this._path
|
|
2873
|
+
});
|
|
2874
|
+
}
|
|
2875
|
+
this._requestedFields = fields === void 0 ? null : fields;
|
|
2876
|
+
this._onProgress = onProgress;
|
|
2877
|
+
this._signal = signal;
|
|
1778
2878
|
this._buffer = null;
|
|
1779
2879
|
this._headerOffset = null;
|
|
1780
2880
|
this._symbolTableOffset = null;
|
|
1781
2881
|
this._indexTableOffset = null;
|
|
1782
2882
|
this._header = null;
|
|
2883
|
+
this._allFields = null;
|
|
2884
|
+
this._selectedFields = null;
|
|
2885
|
+
this._fieldBitMetadataValidated = false;
|
|
1783
2886
|
this._symbolTable = null;
|
|
1784
2887
|
this._indexColumns = null;
|
|
1785
2888
|
this._rowsDecoded = 0;
|
|
2889
|
+
this._bufferFirstRow = 0;
|
|
1786
2890
|
this._fileSize = null;
|
|
1787
2891
|
this._headerMatchesFile = false;
|
|
1788
2892
|
}
|
|
2893
|
+
/**
|
|
2894
|
+
* Emits a progress event if a callback is registered.
|
|
2895
|
+
*
|
|
2896
|
+
* The same shape `QvdFileWriter._emitProgress` emits, deliberately: a caller who has written a
|
|
2897
|
+
* progress bar for a write should not have to write a second one for a read. The stage names
|
|
2898
|
+
* differ because the stages differ, but `symbol-table` and `index-table` mean the same thing on
|
|
2899
|
+
* both sides.
|
|
2900
|
+
*
|
|
2901
|
+
* @param {string} stage The current stage of the read.
|
|
2902
|
+
* @param {number} current The current progress value.
|
|
2903
|
+
* @param {number} total The total progress value.
|
|
2904
|
+
* @private
|
|
2905
|
+
*/
|
|
2906
|
+
_emitProgress(stage, current, total) {
|
|
2907
|
+
if (this._onProgress) {
|
|
2908
|
+
const percent = total > 0 ? Math.round(current / total * 100) : 100;
|
|
2909
|
+
this._onProgress({ stage, current, total, percent });
|
|
2910
|
+
}
|
|
2911
|
+
}
|
|
2912
|
+
/**
|
|
2913
|
+
* Throws if the caller has cancelled the read.
|
|
2914
|
+
*
|
|
2915
|
+
* Throws `signal.reason` - a `DOMException` named `AbortError` unless the caller aborted with a
|
|
2916
|
+
* reason of their own. That is what `AbortSignal` means everywhere else in Node, and inventing
|
|
2917
|
+
* a `QvdAbortError` here would make this library's cancellation the one a caller has to special
|
|
2918
|
+
* case.
|
|
2919
|
+
*
|
|
2920
|
+
* @private
|
|
2921
|
+
*/
|
|
2922
|
+
_throwIfAborted() {
|
|
2923
|
+
if (this._signal) {
|
|
2924
|
+
this._signal.throwIfAborted();
|
|
2925
|
+
}
|
|
2926
|
+
}
|
|
1789
2927
|
/**
|
|
1790
2928
|
* Reads the binary data of the QVD file.
|
|
1791
2929
|
*
|
|
1792
|
-
*
|
|
1793
|
-
*
|
|
2930
|
+
* A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
|
|
2931
|
+
* needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
|
|
2932
|
+
* over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
|
|
1794
2933
|
*
|
|
1795
|
-
*
|
|
1796
|
-
*
|
|
1797
|
-
*
|
|
1798
|
-
*
|
|
2934
|
+
* The saving is in the index table and the rows, not in the symbol table, which is read in
|
|
2935
|
+
* full whatever the window because a stored index in any row can address any symbol. So the
|
|
2936
|
+
* gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
|
|
2937
|
+
* values there is very little to save, which is what `symbolFilteringThreshold` and the
|
|
2938
|
+
* two-pass path exist for.
|
|
1799
2939
|
*
|
|
1800
|
-
* Algorithm for
|
|
2940
|
+
* Algorithm for a windowed read:
|
|
1801
2941
|
* 1. Stream-read the file until XML header delimiter is found
|
|
1802
2942
|
* 2. Parse header to determine symbol table and index table locations
|
|
1803
2943
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
@@ -1810,14 +2950,23 @@ var init_QvdFileReader = __esm({
|
|
|
1810
2950
|
* - Streaming for header finding is efficient for unknown header sizes
|
|
1811
2951
|
* - Direct byte-range reading for remaining data is fastest
|
|
1812
2952
|
*
|
|
1813
|
-
*
|
|
2953
|
+
* A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
|
|
2954
|
+
* table from the front of the file, and the window's records from wherever they sit. The bytes
|
|
2955
|
+
* between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
|
|
2956
|
+
* fixture a 0.4MB read rather than a 38MB one.
|
|
2957
|
+
*
|
|
2958
|
+
* @param {QvdRowWindow} window The rows to read.
|
|
1814
2959
|
* @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
|
|
1815
2960
|
* symbol and index tables on disk. This is the metadata-only path: the header is a few
|
|
1816
2961
|
* kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
|
|
1817
2962
|
* for a 40GB one.
|
|
2963
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
|
|
2964
|
+
* that is fewer than the window covers - see `_prepare`.
|
|
1818
2965
|
* @private
|
|
1819
2966
|
*/
|
|
1820
|
-
async _readData(
|
|
2967
|
+
async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
|
|
2968
|
+
this._throwIfAborted();
|
|
2969
|
+
this._emitProgress("read", 0, 1);
|
|
1821
2970
|
const HEADER_DELIMITER = "\r\n\0";
|
|
1822
2971
|
const CHUNK_SIZE = 64 * 1024;
|
|
1823
2972
|
const stream = fs__default.default.createReadStream(this._path, {
|
|
@@ -1859,6 +3008,8 @@ var init_QvdFileReader = __esm({
|
|
|
1859
3008
|
if (!isExpectedEarlyClose) {
|
|
1860
3009
|
throw error;
|
|
1861
3010
|
}
|
|
3011
|
+
} finally {
|
|
3012
|
+
await closeReadStream(stream);
|
|
1862
3013
|
}
|
|
1863
3014
|
if (headerDelimiterIndex === -1) {
|
|
1864
3015
|
throw new exports.QvdCorruptedError(
|
|
@@ -1887,13 +3038,14 @@ var init_QvdFileReader = __esm({
|
|
|
1887
3038
|
const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
|
|
1888
3039
|
if (headerOnly) {
|
|
1889
3040
|
this._buffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3041
|
+
this._emitProgress("read", 1, 1);
|
|
1890
3042
|
return;
|
|
1891
3043
|
}
|
|
1892
3044
|
let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
|
|
1893
3045
|
if (headerFields && !Array.isArray(headerFields)) {
|
|
1894
3046
|
headerFields = [headerFields];
|
|
1895
3047
|
}
|
|
1896
|
-
const columnCount = Array.isArray(headerFields) ? headerFields.length : 0;
|
|
3048
|
+
const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
|
|
1897
3049
|
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
1898
3050
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
1899
3051
|
);
|
|
@@ -1902,23 +3054,28 @@ var init_QvdFileReader = __esm({
|
|
|
1902
3054
|
this._fileSize = fileSize;
|
|
1903
3055
|
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
|
|
1904
3056
|
}
|
|
3057
|
+
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3058
|
+
const windowRows = resolved.limit;
|
|
1905
3059
|
if (headerNumbersUsable && this._headerMatchesFile) {
|
|
1906
3060
|
validateMemoryAvailability(
|
|
1907
3061
|
symbolTableLength,
|
|
1908
|
-
|
|
3062
|
+
windowRows,
|
|
1909
3063
|
totalRows,
|
|
1910
3064
|
this._path,
|
|
1911
3065
|
this._memorySafetyFactor,
|
|
1912
3066
|
columnCount,
|
|
1913
|
-
this._materialisesRows
|
|
3067
|
+
this._materialisesRows,
|
|
3068
|
+
liveRows
|
|
1914
3069
|
);
|
|
1915
3070
|
}
|
|
1916
|
-
if (
|
|
3071
|
+
if (window.offset === 0 && window.limit === null) {
|
|
1917
3072
|
this._buffer = await fs__default.default.promises.readFile(this._path);
|
|
1918
3073
|
this._fileSize = this._buffer.length;
|
|
3074
|
+
this._bufferFirstRow = 0;
|
|
3075
|
+
this._emitProgress("read", 1, 1);
|
|
1919
3076
|
return;
|
|
1920
3077
|
}
|
|
1921
|
-
const rowsToLoad =
|
|
3078
|
+
const rowsToLoad = windowRows;
|
|
1922
3079
|
validateSymbolTableSizeEarly(symbolTableLength, this._path);
|
|
1923
3080
|
for (const [name, value] of [
|
|
1924
3081
|
["Offset", symbolTableLength],
|
|
@@ -1934,39 +3091,78 @@ var init_QvdFileReader = __esm({
|
|
|
1934
3091
|
});
|
|
1935
3092
|
}
|
|
1936
3093
|
}
|
|
3094
|
+
const skippedIndexBytes = resolved.offset * recordSize;
|
|
1937
3095
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
1938
3096
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
3097
|
+
const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
|
|
1939
3098
|
const fd = await fs__default.default.promises.open(this._path, "r");
|
|
1940
3099
|
try {
|
|
1941
3100
|
const { size: fileSize } = await fd.stat();
|
|
1942
3101
|
this._fileSize = fileSize;
|
|
1943
|
-
if (
|
|
3102
|
+
if (fileBytesRequired > fileSize) {
|
|
1944
3103
|
throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
|
|
1945
3104
|
file: this._path,
|
|
1946
3105
|
fileSize,
|
|
1947
|
-
requiredBytes:
|
|
3106
|
+
requiredBytes: fileBytesRequired,
|
|
1948
3107
|
stage: "readData"
|
|
1949
3108
|
});
|
|
1950
3109
|
}
|
|
1951
3110
|
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
stage: "readData"
|
|
1963
|
-
});
|
|
1964
|
-
}
|
|
1965
|
-
position += bytesRead;
|
|
3111
|
+
await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3112
|
+
if (indexTableBytesToRead > 0) {
|
|
3113
|
+
await this._readRange(
|
|
3114
|
+
fd,
|
|
3115
|
+
indexTableOffset,
|
|
3116
|
+
indexTableBytesToRead,
|
|
3117
|
+
indexTableOffset + skippedIndexBytes,
|
|
3118
|
+
fileSize,
|
|
3119
|
+
fileBytesRequired
|
|
3120
|
+
);
|
|
1966
3121
|
}
|
|
3122
|
+
this._bufferFirstRow = resolved.offset;
|
|
1967
3123
|
} finally {
|
|
1968
3124
|
await fd.close();
|
|
1969
3125
|
}
|
|
3126
|
+
this._emitProgress("read", 1, 1);
|
|
3127
|
+
}
|
|
3128
|
+
/**
|
|
3129
|
+
* Reads one byte range of the file into the buffer.
|
|
3130
|
+
*
|
|
3131
|
+
* Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
|
|
3132
|
+
* 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
|
|
3133
|
+
* try/catch can intercept.
|
|
3134
|
+
*
|
|
3135
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
3136
|
+
* @param {number} bufferOffset Where in the buffer to write.
|
|
3137
|
+
* @param {number} byteCount How many bytes to read.
|
|
3138
|
+
* @param {number} filePosition Where in the file to read from.
|
|
3139
|
+
* @param {number} fileSize The file's size, for the error.
|
|
3140
|
+
* @param {number} requiredBytes Bytes the whole read needs, for the error.
|
|
3141
|
+
* @private
|
|
3142
|
+
*/
|
|
3143
|
+
async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
|
|
3144
|
+
assert2__default.default(this._buffer, "The read buffer has not been allocated.");
|
|
3145
|
+
let done = 0;
|
|
3146
|
+
while (done < byteCount) {
|
|
3147
|
+
const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
|
|
3148
|
+
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
|
|
3149
|
+
if (bytesRead === 0) {
|
|
3150
|
+
throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
|
|
3151
|
+
file: this._path,
|
|
3152
|
+
fileSize,
|
|
3153
|
+
// Two numbers, because they stopped being the same one when a window began reading two
|
|
3154
|
+
// ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
|
|
3155
|
+
// file it gave up. Reporting the position under the name of the count made a windowed
|
|
3156
|
+
// read of a truncated file claim tens of megabytes had been read when a few hundred
|
|
3157
|
+
// bytes had.
|
|
3158
|
+
bytesRead: done,
|
|
3159
|
+
filePosition: filePosition + done,
|
|
3160
|
+
requiredBytes,
|
|
3161
|
+
stage: "readData"
|
|
3162
|
+
});
|
|
3163
|
+
}
|
|
3164
|
+
done += bytesRead;
|
|
3165
|
+
}
|
|
1970
3166
|
}
|
|
1971
3167
|
/**
|
|
1972
3168
|
* Parses the XML header of the QVD file. This method is part of the parsing process
|
|
@@ -2026,6 +3222,8 @@ var init_QvdFileReader = __esm({
|
|
|
2026
3222
|
this._headerOffset = headerBeginIndex;
|
|
2027
3223
|
this._symbolTableOffset = headerEndIndex;
|
|
2028
3224
|
this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3225
|
+
this._allFields = fieldList;
|
|
3226
|
+
this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
|
|
2029
3227
|
}
|
|
2030
3228
|
/**
|
|
2031
3229
|
* Establishes the geometry of the index table, and validates it.
|
|
@@ -2036,14 +3234,15 @@ var init_QvdFileReader = __esm({
|
|
|
2036
3234
|
* about keeping the sign in step with the other one: the two could drift, and #113 is what
|
|
2037
3235
|
* that looks like when they do. There is one copy now.
|
|
2038
3236
|
*
|
|
2039
|
-
* @param {
|
|
3237
|
+
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
2040
3238
|
* @param {string} stage Stage name for any error raised here.
|
|
2041
3239
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
2042
|
-
* indexBuffer: Buffer}} The record geometry.
|
|
3240
|
+
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
|
|
3241
|
+
* record, so the decoder always counts from zero.
|
|
2043
3242
|
* @private
|
|
2044
3243
|
*/
|
|
2045
|
-
_planIndexTable(
|
|
2046
|
-
if (!this._buffer || !this._header || !this._indexTableOffset) {
|
|
3244
|
+
_planIndexTable(window, stage) {
|
|
3245
|
+
if (!this._buffer || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
2047
3246
|
throw new exports.QvdCorruptedError(
|
|
2048
3247
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
2049
3248
|
{
|
|
@@ -2052,14 +3251,12 @@ var init_QvdFileReader = __esm({
|
|
|
2052
3251
|
}
|
|
2053
3252
|
);
|
|
2054
3253
|
}
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
fields = [fields];
|
|
2058
|
-
}
|
|
3254
|
+
const allFields = this._allFields;
|
|
3255
|
+
const fields = this._selectedFields;
|
|
2059
3256
|
const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
|
|
2060
3257
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
2061
|
-
const rowsToLoad = rowLimit !== null ? Math.min(rowLimit, totalRows) : totalRows;
|
|
2062
3258
|
const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
|
|
3259
|
+
const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
|
|
2063
3260
|
validateIndexTableMetadata(
|
|
2064
3261
|
recordSize,
|
|
2065
3262
|
totalRows,
|
|
@@ -2068,11 +3265,20 @@ var init_QvdFileReader = __esm({
|
|
|
2068
3265
|
this._buffer.length,
|
|
2069
3266
|
rowsToLoad,
|
|
2070
3267
|
this._path,
|
|
2071
|
-
this._fileSize
|
|
3268
|
+
this._fileSize,
|
|
3269
|
+
firstRow,
|
|
3270
|
+
this._bufferFirstRow
|
|
2072
3271
|
);
|
|
2073
|
-
const
|
|
2074
|
-
|
|
2075
|
-
|
|
3272
|
+
const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
|
|
3273
|
+
const indexBuffer = this._buffer.subarray(
|
|
3274
|
+
this._indexTableOffset + bufferRecordStart,
|
|
3275
|
+
this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
|
|
3276
|
+
);
|
|
3277
|
+
if (!this._fieldBitMetadataValidated) {
|
|
3278
|
+
for (const field of allFields) {
|
|
3279
|
+
validateFieldBitMetadata(field, recordSize, this._path);
|
|
3280
|
+
}
|
|
3281
|
+
this._fieldBitMetadataValidated = true;
|
|
2076
3282
|
}
|
|
2077
3283
|
assert2__default.default(
|
|
2078
3284
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
@@ -2084,31 +3290,45 @@ var init_QvdFileReader = __esm({
|
|
|
2084
3290
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
2085
3291
|
* This is used for two-pass symbol filtering optimization.
|
|
2086
3292
|
*
|
|
2087
|
-
*
|
|
2088
|
-
*
|
|
3293
|
+
* Only the selected fields are analysed. An unselected field's symbols are never parsed, so
|
|
3294
|
+
* there is nothing for a usage set to filter and decoding its column would be a pass over the
|
|
3295
|
+
* whole window for an answer nobody reads.
|
|
3296
|
+
*
|
|
3297
|
+
* @param {QvdRowWindow} window The rows to analyse.
|
|
3298
|
+
* @return {Promise<Array<Set<number>>>} One set of needed symbol indices per selected field, in
|
|
3299
|
+
* the same order `_parseSymbolTable` walks them.
|
|
2089
3300
|
* @private
|
|
2090
3301
|
*/
|
|
2091
|
-
async _analyzeIndexTableSymbolUsage(
|
|
2092
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(
|
|
2093
|
-
const symbolUsage =
|
|
2094
|
-
const
|
|
2095
|
-
|
|
3302
|
+
async _analyzeIndexTableSymbolUsage(window) {
|
|
3303
|
+
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
|
|
3304
|
+
const symbolUsage = [];
|
|
3305
|
+
const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
|
|
3306
|
+
const column = new Int32Array(sliceRows);
|
|
3307
|
+
fields.forEach((field, position) => {
|
|
3308
|
+
this._throwIfAborted();
|
|
2096
3309
|
const needed = /* @__PURE__ */ new Set();
|
|
2097
|
-
symbolUsage
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
3310
|
+
symbolUsage[position] = needed;
|
|
3311
|
+
const bitOffset = parseInt(field["BitOffset"], 10);
|
|
3312
|
+
const bitWidth = parseInt(field["BitWidth"], 10);
|
|
3313
|
+
const bias = parseInt(field["Bias"], 10);
|
|
3314
|
+
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3315
|
+
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3316
|
+
decodeIndexColumn(
|
|
3317
|
+
first === 0 ? indexBuffer : indexBuffer.subarray(first * recordSize),
|
|
3318
|
+
recordSize,
|
|
3319
|
+
count,
|
|
3320
|
+
bitOffset,
|
|
3321
|
+
bitWidth,
|
|
3322
|
+
bias,
|
|
3323
|
+
column
|
|
3324
|
+
);
|
|
3325
|
+
for (let row = 0; row < count; row++) {
|
|
3326
|
+
if (column[row] >= 0) {
|
|
3327
|
+
needed.add(column[row]);
|
|
3328
|
+
}
|
|
2110
3329
|
}
|
|
2111
3330
|
}
|
|
3331
|
+
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
2112
3332
|
});
|
|
2113
3333
|
return symbolUsage;
|
|
2114
3334
|
}
|
|
@@ -2116,12 +3336,20 @@ var init_QvdFileReader = __esm({
|
|
|
2116
3336
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
2117
3337
|
* and should not be called directly.
|
|
2118
3338
|
*
|
|
2119
|
-
*
|
|
2120
|
-
*
|
|
2121
|
-
*
|
|
3339
|
+
* A field the caller did not select is skipped whole. Its symbol area is neither scanned nor
|
|
3340
|
+
* parsed - the per-field `Offset` and `Length` say exactly where it is, so there is nothing to
|
|
3341
|
+
* walk past - and that is where field selection earns its keep. The index decode is cheap by
|
|
3342
|
+
* comparison; parsing symbols is not.
|
|
3343
|
+
*
|
|
3344
|
+
* @param {Array<Set<number>>|null} symbolsToKeep Optional set of symbol indices to keep per
|
|
3345
|
+
* selected field, indexed by position. If provided, only these symbols will be parsed
|
|
3346
|
+
* (two-pass filtering optimization).
|
|
3347
|
+
* @param {number} rowsToLoad Rows the read covers, for memory estimation.
|
|
3348
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
|
|
3349
|
+
* that is fewer than the window covers - see `_prepare`.
|
|
2122
3350
|
*/
|
|
2123
|
-
async _parseSymbolTable(symbolsToKeep = null,
|
|
2124
|
-
if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset) {
|
|
3351
|
+
async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
|
|
3352
|
+
if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
2125
3353
|
throw new exports.QvdCorruptedError(
|
|
2126
3354
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
2127
3355
|
{
|
|
@@ -2130,7 +3358,8 @@ var init_QvdFileReader = __esm({
|
|
|
2130
3358
|
}
|
|
2131
3359
|
);
|
|
2132
3360
|
}
|
|
2133
|
-
|
|
3361
|
+
const allFields = this._allFields;
|
|
3362
|
+
const fields = this._selectedFields;
|
|
2134
3363
|
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
2135
3364
|
const symbolTableSize = symbolBuffer.length;
|
|
2136
3365
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
@@ -2138,51 +3367,34 @@ var init_QvdFileReader = __esm({
|
|
|
2138
3367
|
if (this._headerMatchesFile) {
|
|
2139
3368
|
validateMemoryAvailability(
|
|
2140
3369
|
symbolTableSize,
|
|
2141
|
-
|
|
3370
|
+
rowsToLoad,
|
|
2142
3371
|
totalRows,
|
|
2143
3372
|
this._path,
|
|
2144
3373
|
this._memorySafetyFactor,
|
|
2145
|
-
|
|
2146
|
-
this._materialisesRows
|
|
3374
|
+
fields.length,
|
|
3375
|
+
this._materialisesRows,
|
|
3376
|
+
liveRows
|
|
2147
3377
|
);
|
|
2148
3378
|
}
|
|
2149
|
-
warnLargeSymbolTable(
|
|
2150
|
-
|
|
2151
|
-
maxRows,
|
|
2152
|
-
totalRows,
|
|
2153
|
-
Array.isArray(fields) ? fields.length : 1,
|
|
2154
|
-
this._materialisesRows
|
|
2155
|
-
);
|
|
2156
|
-
if (!Array.isArray(fields)) {
|
|
2157
|
-
fields = [fields];
|
|
2158
|
-
}
|
|
2159
|
-
for (const field of fields) {
|
|
3379
|
+
warnLargeSymbolTable(symbolTableSize, rowsToLoad, totalRows, fields.length, this._materialisesRows);
|
|
3380
|
+
for (const field of allFields) {
|
|
2160
3381
|
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
2161
3382
|
}
|
|
2162
|
-
this._symbolTable = fields.map((field) => {
|
|
3383
|
+
this._symbolTable = fields.map((field, position) => {
|
|
3384
|
+
this._throwIfAborted();
|
|
2163
3385
|
const symbolsOffset = parseInt(field["Offset"], 10);
|
|
2164
3386
|
const symbolsLength = parseInt(field["Length"], 10);
|
|
2165
|
-
const
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
pointer,
|
|
2177
|
-
symbolBuffer.length,
|
|
2178
|
-
fieldName,
|
|
2179
|
-
this._path,
|
|
2180
|
-
shouldKeepSymbol
|
|
2181
|
-
);
|
|
2182
|
-
symbols.push(symbol);
|
|
2183
|
-
pointer += bytesRead - 1;
|
|
2184
|
-
symbolIndex++;
|
|
2185
|
-
}
|
|
3387
|
+
const symbols = parseFieldSymbols(
|
|
3388
|
+
symbolBuffer,
|
|
3389
|
+
symbolsOffset,
|
|
3390
|
+
symbolsOffset + symbolsLength,
|
|
3391
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
3392
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
3393
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
3394
|
+
field["FieldName"],
|
|
3395
|
+
this._path
|
|
3396
|
+
);
|
|
3397
|
+
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
2186
3398
|
return symbols;
|
|
2187
3399
|
});
|
|
2188
3400
|
}
|
|
@@ -2201,13 +3413,18 @@ var init_QvdFileReader = __esm({
|
|
|
2201
3413
|
* same for every row, so they are hoisted out of the loop and the inner loop does arithmetic
|
|
2202
3414
|
* into a typed array and nothing else. Rows are assembled later, once, in `load()`.
|
|
2203
3415
|
*
|
|
2204
|
-
*
|
|
3416
|
+
* The window is what makes chunked iteration cheap: `decodeIndexColumn` walks records by
|
|
3417
|
+
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3418
|
+
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3419
|
+
*
|
|
3420
|
+
* @param {QvdRowWindow} window The rows to decode.
|
|
2205
3421
|
*/
|
|
2206
|
-
async _parseIndexTable(
|
|
2207
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(
|
|
3422
|
+
async _parseIndexTable(window) {
|
|
3423
|
+
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
2208
3424
|
this._rowsDecoded = rowsToLoad;
|
|
2209
|
-
this._indexColumns = fields.map(
|
|
2210
|
-
(
|
|
3425
|
+
this._indexColumns = fields.map((field, position) => {
|
|
3426
|
+
this._throwIfAborted();
|
|
3427
|
+
const column = decodeIndexColumn(
|
|
2211
3428
|
indexBuffer,
|
|
2212
3429
|
recordSize,
|
|
2213
3430
|
rowsToLoad,
|
|
@@ -2215,8 +3432,10 @@ var init_QvdFileReader = __esm({
|
|
|
2215
3432
|
parseInt(field["BitWidth"], 10),
|
|
2216
3433
|
parseInt(field["Bias"], 10),
|
|
2217
3434
|
new Int32Array(rowsToLoad)
|
|
2218
|
-
)
|
|
2219
|
-
|
|
3435
|
+
);
|
|
3436
|
+
this._emitProgress("index-table", position + 1, fields.length);
|
|
3437
|
+
return column;
|
|
3438
|
+
});
|
|
2220
3439
|
}
|
|
2221
3440
|
/**
|
|
2222
3441
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -2233,8 +3452,11 @@ var init_QvdFileReader = __esm({
|
|
|
2233
3452
|
* @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
|
|
2234
3453
|
*/
|
|
2235
3454
|
async loadMetadata() {
|
|
2236
|
-
await this._readData(null, true);
|
|
3455
|
+
await this._readData({ offset: 0, limit: null }, true);
|
|
3456
|
+
this._emitProgress("header", 0, 1);
|
|
2237
3457
|
await this._parseHeader();
|
|
3458
|
+
this._emitProgress("header", 1, 1);
|
|
3459
|
+
this._throwIfAborted();
|
|
2238
3460
|
assert2__default.default(this._header, "The QVD file header has not been parsed.");
|
|
2239
3461
|
const header = this._header["QvdTableHeader"];
|
|
2240
3462
|
let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
|
|
@@ -2269,137 +3491,337 @@ var init_QvdFileReader = __esm({
|
|
|
2269
3491
|
/**
|
|
2270
3492
|
* Loads the QVD file into memory and parses it.
|
|
2271
3493
|
*
|
|
2272
|
-
* @param {number|null
|
|
2273
|
-
*
|
|
2274
|
-
*
|
|
3494
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3495
|
+
* The rows to load. A number or null means what it always meant - the first N rows, or all of
|
|
3496
|
+
* them - and `{offset, limit}` is the same thing said more precisely, so `5` and
|
|
3497
|
+
* `{offset: 0, limit: 5}` are one read. `maxRows` is accepted as a second name for `limit`.
|
|
3498
|
+
* @throws {QvdValidationError} If the window is not a non-negative integer, null, or a valid
|
|
3499
|
+
* `{offset, limit}` object.
|
|
2275
3500
|
* @return {Promise<QvdDataFrame>} The loaded QVD file.
|
|
2276
3501
|
*/
|
|
2277
|
-
async load(
|
|
2278
|
-
const
|
|
3502
|
+
async load(window = null) {
|
|
3503
|
+
const rows = normaliseWindow(window, this._path);
|
|
3504
|
+
const prepared = await this._prepare(rows);
|
|
3505
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3506
|
+
const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
|
|
3507
|
+
return new exports.QvdDataFrame(
|
|
3508
|
+
data,
|
|
3509
|
+
prepared.columns,
|
|
3510
|
+
prepared.metadata,
|
|
3511
|
+
{
|
|
3512
|
+
...prepared.loadStats,
|
|
3513
|
+
rowsLoaded: data.length
|
|
3514
|
+
},
|
|
3515
|
+
prepared.storedSymbols
|
|
3516
|
+
);
|
|
3517
|
+
}
|
|
3518
|
+
/**
|
|
3519
|
+
* Reads the file as columns, without ever materialising rows.
|
|
3520
|
+
*
|
|
3521
|
+
* Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
|
|
3522
|
+
* What it keeps instead is what the decoder already produced: one `Int32Array` of stored
|
|
3523
|
+
* indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
|
|
3524
|
+
* fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
|
|
3525
|
+
* bytes per row rather than a boxed value per cell, and the symbols are a few thousand
|
|
3526
|
+
* entries shared across every row that uses them.
|
|
3527
|
+
*
|
|
3528
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3529
|
+
* The rows to decode, in the same spellings `load()` accepts.
|
|
3530
|
+
* @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
|
|
3531
|
+
*/
|
|
3532
|
+
async loadColumnar(window = null) {
|
|
3533
|
+
const rows = normaliseWindow(window, this._path);
|
|
3534
|
+
const prepared = await this._prepare(rows, null, true);
|
|
3535
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3536
|
+
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
2279
3537
|
assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
}
|
|
2289
|
-
|
|
2290
|
-
}
|
|
2291
|
-
loadStats.rowsLoaded = data.length;
|
|
2292
|
-
return new exports.QvdDataFrame(data, columns, metadata, loadStats);
|
|
3538
|
+
return new QvdColumnTable2({
|
|
3539
|
+
columns: prepared.columns,
|
|
3540
|
+
codesByField: this._indexColumns,
|
|
3541
|
+
symbolsByField: prepared.resolvedByField,
|
|
3542
|
+
halvesByField: prepared.halvesByField,
|
|
3543
|
+
rowCount: this._rowsDecoded,
|
|
3544
|
+
metadata: prepared.metadata,
|
|
3545
|
+
storedSymbols: prepared.storedSymbols,
|
|
3546
|
+
loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
|
|
3547
|
+
});
|
|
2293
3548
|
}
|
|
2294
3549
|
/**
|
|
2295
|
-
*
|
|
3550
|
+
* Yields the window as data frames of at most `chunkSize` rows.
|
|
2296
3551
|
*
|
|
2297
|
-
*
|
|
2298
|
-
*
|
|
2299
|
-
*
|
|
2300
|
-
*
|
|
2301
|
-
*
|
|
3552
|
+
* The file is opened, read and parsed **once**; only the index decode and the row building
|
|
3553
|
+
* happen per chunk. That is the whole reason this exists as a method rather than as a loop of
|
|
3554
|
+
* `load({offset, limit})` calls at the call site: the symbol table has to be parsed in full
|
|
3555
|
+
* whatever the chunk size - a stored index in the last chunk can address the first symbol -
|
|
3556
|
+
* and re-parsing it per chunk is what makes the obvious implementation cost more than a plain
|
|
3557
|
+
* load rather than less. PyQvd's chunked read does re-read it, and the comment on #140 records
|
|
3558
|
+
* that as a limitation rather than a design.
|
|
2302
3559
|
*
|
|
2303
|
-
*
|
|
2304
|
-
*
|
|
2305
|
-
*
|
|
2306
|
-
*
|
|
3560
|
+
* What it bounds is row materialisation, which is what actually dominates a large read's heap.
|
|
3561
|
+
* Two chunks of rows are alive at a time, not one - `for await` keeps the yielded frame
|
|
3562
|
+
* reachable while this generator builds the next - which is why `liveRows` below is
|
|
3563
|
+
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
3564
|
+
*
|
|
3565
|
+
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
3566
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
3567
|
+
*
|
|
3568
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
3569
|
+
* The rows to cover, in the same spellings `load()` accepts.
|
|
3570
|
+
* @param {number} chunkSize Rows per frame. Must be a positive integer.
|
|
3571
|
+
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
|
|
2307
3572
|
*/
|
|
2308
|
-
async
|
|
2309
|
-
if (
|
|
2310
|
-
throw new exports.QvdValidationError("
|
|
2311
|
-
provided:
|
|
2312
|
-
type: typeof
|
|
3573
|
+
async *iterateRows(window, chunkSize) {
|
|
3574
|
+
if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
3575
|
+
throw new exports.QvdValidationError("chunkSize must be a positive integer", {
|
|
3576
|
+
provided: chunkSize,
|
|
3577
|
+
type: typeof chunkSize,
|
|
2313
3578
|
file: this._path
|
|
2314
3579
|
});
|
|
2315
3580
|
}
|
|
2316
|
-
|
|
3581
|
+
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
3582
|
+
const rows = normaliseWindow(window, this._path);
|
|
3583
|
+
const prepared = await this._prepare(rows, liveRows);
|
|
3584
|
+
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
3585
|
+
this._throwIfAborted();
|
|
3586
|
+
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
3587
|
+
const offset = prepared.offset + done;
|
|
3588
|
+
await this._parseIndexTable({ offset, limit: count });
|
|
3589
|
+
const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
|
|
3590
|
+
yield new exports.QvdDataFrame(
|
|
3591
|
+
data,
|
|
3592
|
+
prepared.columns,
|
|
3593
|
+
prepared.metadata,
|
|
3594
|
+
{
|
|
3595
|
+
...prepared.loadStats,
|
|
3596
|
+
offset,
|
|
3597
|
+
rowsLoaded: data.length
|
|
3598
|
+
},
|
|
3599
|
+
prepared.storedSymbols
|
|
3600
|
+
);
|
|
3601
|
+
}
|
|
3602
|
+
}
|
|
3603
|
+
/**
|
|
3604
|
+
* Reads the file and resolves its symbols, stopping short of decoding any rows.
|
|
3605
|
+
*
|
|
3606
|
+
* Everything `load()`, `loadColumnar()` and `iterateRows()` have in common, which is everything
|
|
3607
|
+
* that depends on the file rather than on the window. Two read paths for one binary format is
|
|
3608
|
+
* the drift risk #113 is the standing example of - a stored index resolved one way here and
|
|
3609
|
+
* another way there returns plausible wrong values and throws nothing - so there is one path,
|
|
3610
|
+
* and the entry points differ only in what they do with what it returns and how many rows they
|
|
3611
|
+
* ask for at a time.
|
|
3612
|
+
*
|
|
3613
|
+
* @param {QvdRowWindow} window The rows the read covers.
|
|
3614
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
|
|
3615
|
+
* is fewer than the window covers, and how many of them one row of the caller's chunk size
|
|
3616
|
+
* accounts for. Only `iterateRows` passes it; every other read holds what it covers.
|
|
3617
|
+
* @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
|
|
3618
|
+
* cells do not show them, which only a columnar read has a use for.
|
|
3619
|
+
* @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
|
|
3620
|
+
* resolvedByField: Array<Array<any>>,
|
|
3621
|
+
* halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
|
|
3622
|
+
* storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
|
|
3623
|
+
* rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
|
|
3624
|
+
* against it.
|
|
3625
|
+
* @private
|
|
3626
|
+
*/
|
|
3627
|
+
async _prepare(window, liveRows = null, wantHalves = false) {
|
|
3628
|
+
this._throwIfAborted();
|
|
3629
|
+
await this._readData(window, false, liveRows);
|
|
3630
|
+
this._emitProgress("header", 0, 1);
|
|
2317
3631
|
await this._parseHeader();
|
|
3632
|
+
this._emitProgress("header", 1, 1);
|
|
3633
|
+
this._throwIfAborted();
|
|
3634
|
+
assert2__default.default(this._header, "The QVD file header has not been parsed.");
|
|
3635
|
+
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
3636
|
+
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3637
|
+
const resolved = resolveWindow(window, totalRows);
|
|
3638
|
+
const rowsAvailable = resolved.limit;
|
|
2318
3639
|
let symbolsToKeep = null;
|
|
2319
3640
|
let symbolsKept = null;
|
|
2320
|
-
if (
|
|
2321
|
-
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3641
|
+
if (window.limit !== null || window.offset > 0) {
|
|
2322
3642
|
if (symbolTableLength > this._symbolFilteringThreshold) {
|
|
2323
|
-
symbolsToKeep = await this._analyzeIndexTableSymbolUsage(
|
|
2324
|
-
symbolsKept =
|
|
3643
|
+
symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
|
|
3644
|
+
symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
|
|
2325
3645
|
}
|
|
2326
3646
|
}
|
|
2327
|
-
await this._parseSymbolTable(symbolsToKeep,
|
|
2328
|
-
await this._parseIndexTable(maxRows);
|
|
2329
|
-
assert2__default.default(this._header, "The QVD file header has not been parsed.");
|
|
3647
|
+
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
2330
3648
|
assert2__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
3649
|
+
this._throwIfAborted();
|
|
3650
|
+
assert2__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3651
|
+
const resolvedByField = [];
|
|
3652
|
+
const halvesByField = [];
|
|
3653
|
+
const entries = [];
|
|
3654
|
+
this._symbolTable.forEach((symbols, position) => {
|
|
3655
|
+
const { values, entry, halves } = resolveFieldSymbols(
|
|
3656
|
+
symbols,
|
|
3657
|
+
// @ts-ignore - asserted above
|
|
3658
|
+
this._selectedFields[position]["FieldName"],
|
|
3659
|
+
this._duals,
|
|
3660
|
+
this._coerceNumericStrings,
|
|
3661
|
+
wantHalves
|
|
3662
|
+
);
|
|
3663
|
+
resolvedByField.push(values);
|
|
3664
|
+
halvesByField.push(halves);
|
|
3665
|
+
if (entry !== null) {
|
|
3666
|
+
entries.push(entry);
|
|
2337
3667
|
}
|
|
2338
|
-
return resolved;
|
|
2339
3668
|
});
|
|
2340
|
-
|
|
2341
|
-
if (!Array.isArray(fields)) {
|
|
2342
|
-
fields = [fields];
|
|
2343
|
-
}
|
|
2344
|
-
const columns = fields.map((field) => field["FieldName"]);
|
|
3669
|
+
const columns = this._selectedFields.map((field) => field["FieldName"]);
|
|
2345
3670
|
const metadata = this._header["QvdTableHeader"];
|
|
3671
|
+
const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
|
|
3672
|
+
if (storedSymbols !== null) {
|
|
3673
|
+
attachStoredSymbols(metadata, storedSymbols);
|
|
3674
|
+
}
|
|
2346
3675
|
const loadStats = {
|
|
2347
|
-
symbolTableBytes:
|
|
2348
|
-
totalRows
|
|
2349
|
-
rowsLoaded:
|
|
3676
|
+
symbolTableBytes: symbolTableLength,
|
|
3677
|
+
totalRows,
|
|
3678
|
+
rowsLoaded: 0,
|
|
3679
|
+
offset: resolved.offset,
|
|
2350
3680
|
symbolFiltering: symbolsToKeep !== null,
|
|
2351
3681
|
symbolsKept
|
|
2352
3682
|
};
|
|
2353
|
-
return {
|
|
3683
|
+
return {
|
|
3684
|
+
columns,
|
|
3685
|
+
metadata,
|
|
3686
|
+
loadStats,
|
|
3687
|
+
resolvedByField,
|
|
3688
|
+
halvesByField,
|
|
3689
|
+
storedSymbols,
|
|
3690
|
+
rowsAvailable,
|
|
3691
|
+
offset: resolved.offset
|
|
3692
|
+
};
|
|
2354
3693
|
}
|
|
2355
3694
|
/**
|
|
2356
|
-
*
|
|
3695
|
+
* Builds rows from the columns currently decoded.
|
|
2357
3696
|
*
|
|
2358
|
-
*
|
|
2359
|
-
*
|
|
2360
|
-
*
|
|
2361
|
-
*
|
|
2362
|
-
*
|
|
2363
|
-
* entries shared across every row that uses them.
|
|
3697
|
+
* `data` stays eager: of the four ways this library is used - a full read, a preview already
|
|
3698
|
+
* bounded by a limit, writing an array out, and reading metadata - not one is helped by
|
|
3699
|
+
* materialising a row only when it is touched, and a lazy accessor would cost a proxy, a cache
|
|
3700
|
+
* and mutation semantics to serve none of them. A caller who wants columns without paying for
|
|
3701
|
+
* rows uses `QvdColumnTable`, which stops before this loop.
|
|
2364
3702
|
*
|
|
2365
|
-
* @param {
|
|
2366
|
-
* @
|
|
3703
|
+
* @param {Array<Array<any>>} resolvedByField One resolved value per distinct symbol, per field.
|
|
3704
|
+
* @param {number} progressBase Rows already delivered before this call, so that progress over a
|
|
3705
|
+
* chunked iteration counts the whole window rather than restarting at every chunk.
|
|
3706
|
+
* @param {number} progressTotal Rows the whole window covers.
|
|
3707
|
+
* @return {Array<Array<any>>} The rows.
|
|
3708
|
+
* @private
|
|
2367
3709
|
*/
|
|
2368
|
-
|
|
2369
|
-
const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
|
|
2370
|
-
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
3710
|
+
_buildRows(resolvedByField, progressBase, progressTotal) {
|
|
2371
3711
|
assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
3712
|
+
const indexColumns = this._indexColumns;
|
|
3713
|
+
const fieldCount = indexColumns.length;
|
|
3714
|
+
const rowCount = this._rowsDecoded;
|
|
3715
|
+
const data = new Array(rowCount);
|
|
3716
|
+
const reportInterval = Math.max(1, Math.floor(progressTotal / 100));
|
|
3717
|
+
for (let row = 0; row < rowCount; row++) {
|
|
3718
|
+
const values = new Array(fieldCount);
|
|
3719
|
+
for (let field = 0; field < fieldCount; field++) {
|
|
3720
|
+
const symbolIndex = indexColumns[field][row];
|
|
3721
|
+
values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
|
|
3722
|
+
}
|
|
3723
|
+
data[row] = values;
|
|
3724
|
+
if ((progressBase + row + 1) % reportInterval === 0 || row + 1 === rowCount) {
|
|
3725
|
+
this._throwIfAborted();
|
|
3726
|
+
this._emitProgress("rows", progressBase + row + 1, progressTotal);
|
|
3727
|
+
}
|
|
3728
|
+
}
|
|
3729
|
+
return data;
|
|
2380
3730
|
}
|
|
2381
3731
|
};
|
|
2382
3732
|
}
|
|
2383
3733
|
});
|
|
2384
3734
|
|
|
2385
3735
|
// src/QvdDataFrame.js
|
|
3736
|
+
function defaultFieldHeader(fieldName) {
|
|
3737
|
+
return {
|
|
3738
|
+
FieldName: fieldName,
|
|
3739
|
+
BitOffset: 0,
|
|
3740
|
+
BitWidth: 0,
|
|
3741
|
+
Bias: 0,
|
|
3742
|
+
NoOfSymbols: 0,
|
|
3743
|
+
Offset: 0,
|
|
3744
|
+
Length: 0,
|
|
3745
|
+
Comment: "",
|
|
3746
|
+
NumberFormat: {
|
|
3747
|
+
Type: "UNKNOWN",
|
|
3748
|
+
nDec: "0",
|
|
3749
|
+
UseThou: "0",
|
|
3750
|
+
Fmt: "",
|
|
3751
|
+
Dec: "",
|
|
3752
|
+
Thou: ""
|
|
3753
|
+
},
|
|
3754
|
+
Tags: {}
|
|
3755
|
+
};
|
|
3756
|
+
}
|
|
3757
|
+
function defaultHeader(columns) {
|
|
3758
|
+
return {
|
|
3759
|
+
QvBuildNo: 50667,
|
|
3760
|
+
CreatorDoc: "",
|
|
3761
|
+
CreateUtcTime: "",
|
|
3762
|
+
SourceCreateUtcTime: "",
|
|
3763
|
+
SourceFileUtcTime: "",
|
|
3764
|
+
SourceFileSize: -1,
|
|
3765
|
+
StaleUtcTime: "",
|
|
3766
|
+
TableName: "",
|
|
3767
|
+
Fields: {
|
|
3768
|
+
QvdFieldHeader: columns.map(defaultFieldHeader)
|
|
3769
|
+
},
|
|
3770
|
+
NoOfRecords: 0,
|
|
3771
|
+
RecordByteSize: 0,
|
|
3772
|
+
Offset: 0,
|
|
3773
|
+
Length: 0,
|
|
3774
|
+
Compression: "",
|
|
3775
|
+
Comment: "",
|
|
3776
|
+
EncryptionInfo: "",
|
|
3777
|
+
TableTags: "",
|
|
3778
|
+
ProfilingData: "",
|
|
3779
|
+
Lineage: {}
|
|
3780
|
+
};
|
|
3781
|
+
}
|
|
2386
3782
|
exports.QvdDataFrame = void 0;
|
|
2387
3783
|
var init_QvdDataFrame = __esm({
|
|
2388
3784
|
"src/QvdDataFrame.js"() {
|
|
2389
3785
|
init_QvdErrors();
|
|
3786
|
+
init_cellRules();
|
|
3787
|
+
init_readOptions();
|
|
3788
|
+
init_storedSymbols();
|
|
3789
|
+
__name(defaultFieldHeader, "defaultFieldHeader");
|
|
3790
|
+
__name(defaultHeader, "defaultHeader");
|
|
2390
3791
|
exports.QvdDataFrame = class _QvdDataFrame {
|
|
3792
|
+
static {
|
|
3793
|
+
__name(this, "QvdDataFrame");
|
|
3794
|
+
}
|
|
2391
3795
|
/**
|
|
2392
3796
|
* Represents the data frame stored inside a QVD file.
|
|
3797
|
+
*
|
|
3798
|
+
* The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
|
|
3799
|
+
* its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
|
|
3800
|
+
* Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
|
|
3801
|
+
* holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
|
|
3802
|
+
* header's record did for a frame built from some of a read's columns.
|
|
3803
|
+
*
|
|
2393
3804
|
* @param {Array<Array<any>>} data The data of the data frame.
|
|
2394
3805
|
* @param {Array<string>} columns The columns of the data frame.
|
|
2395
3806
|
* @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
|
|
2396
3807
|
* @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
|
|
3808
|
+
* @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
|
|
3809
|
+
* shows only one half of its symbol (optional) - see `storedSymbols`.
|
|
3810
|
+
* @throws {QvdValidationError} If the record is malformed.
|
|
2397
3811
|
*/
|
|
2398
|
-
constructor(data, columns, metadata = null, loadStats = null) {
|
|
3812
|
+
constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
|
|
2399
3813
|
this._data = data;
|
|
2400
3814
|
this._columns = columns;
|
|
2401
3815
|
this._metadata = metadata;
|
|
3816
|
+
this._ownsMetadata = false;
|
|
2402
3817
|
this._loadStats = loadStats;
|
|
3818
|
+
this._storedSymbols = narrowStoredSymbols(
|
|
3819
|
+
normaliseStoredSymbols(
|
|
3820
|
+
// @ts-ignore - a symbol-keyed property the reader defines on the header object
|
|
3821
|
+
storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
|
|
3822
|
+
),
|
|
3823
|
+
columns
|
|
3824
|
+
);
|
|
2403
3825
|
}
|
|
2404
3826
|
/**
|
|
2405
3827
|
* Returns the data of the data frame.
|
|
@@ -2426,15 +3848,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2426
3848
|
get metadata() {
|
|
2427
3849
|
return this._metadata;
|
|
2428
3850
|
}
|
|
3851
|
+
/**
|
|
3852
|
+
* What the frame's cells were read from, where a cell shows only one half of its symbol.
|
|
3853
|
+
*
|
|
3854
|
+
* A dual read as its number has a text the cell does not show; one read as its text has a number;
|
|
3855
|
+
* a string read as a number has the text it was spelled with. The record keeps those halves, per
|
|
3856
|
+
* field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
|
|
3857
|
+
* and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
|
|
3858
|
+
* `select`, `toDict` and `fromDict`.
|
|
3859
|
+
*
|
|
3860
|
+
* Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
|
|
3861
|
+
* the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
|
|
3862
|
+
* pure number.
|
|
3863
|
+
*
|
|
3864
|
+
* @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
|
|
3865
|
+
* showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
|
|
3866
|
+
* in the record it was given or found - one from `select`, or one built from some of a read's
|
|
3867
|
+
* columns and its header - has an empty record rather than null, because a frame given null takes
|
|
3868
|
+
* the record its header carries, which describes every field of the read.
|
|
3869
|
+
*/
|
|
3870
|
+
get storedSymbols() {
|
|
3871
|
+
return this._storedSymbols;
|
|
3872
|
+
}
|
|
2429
3873
|
/**
|
|
2430
3874
|
* Returns statistics about the read that produced this data frame.
|
|
2431
3875
|
*
|
|
2432
|
-
*
|
|
2433
|
-
*
|
|
3876
|
+
* Carried by every frame that came from a file - `fromQvd()`, and each chunk `iterate()` yields,
|
|
3877
|
+
* which is how a chunk reports its `offset`. `fromDict()`, `head()`, `tail()`, `rows()` and
|
|
3878
|
+
* `select()` describe no particular read and report null rather than a stale figure.
|
|
2434
3879
|
*
|
|
2435
3880
|
* The main use is confirming that a lazy load actually filtered the symbol table:
|
|
2436
3881
|
* `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
|
|
2437
|
-
* survived it
|
|
3882
|
+
* survived it. Note that a bounded read does not filter on its own - the two-pass path engages
|
|
3883
|
+
* only above `symbolFilteringThreshold`, so on a file below it this reports false and every
|
|
3884
|
+
* symbol was parsed however few rows were asked for.
|
|
2438
3885
|
*
|
|
2439
3886
|
* @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
|
|
2440
3887
|
*/
|
|
@@ -2543,54 +3990,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2543
3990
|
* @property {string} [profilingData] - Profiling data
|
|
2544
3991
|
* @property {Object|string} [lineage] - Lineage
|
|
2545
3992
|
*/
|
|
3993
|
+
/**
|
|
3994
|
+
* The header a metadata setter may change: this frame's own.
|
|
3995
|
+
*
|
|
3996
|
+
* A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
|
|
3997
|
+
* `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
|
|
3998
|
+
* change copies it, and a change made through one frame never reaches another. A frame with no header
|
|
3999
|
+
* gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
|
|
4000
|
+
* so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
|
|
4001
|
+
*
|
|
4002
|
+
* @return {any} The header.
|
|
4003
|
+
*/
|
|
4004
|
+
_ownMetadata() {
|
|
4005
|
+
if (!this._metadata) {
|
|
4006
|
+
this._metadata = defaultHeader(this._columns);
|
|
4007
|
+
} else if (!this._ownsMetadata) {
|
|
4008
|
+
const record = this._metadata[STORED_SYMBOLS];
|
|
4009
|
+
this._metadata = structuredClone(this._metadata);
|
|
4010
|
+
if (record) {
|
|
4011
|
+
attachStoredSymbols(this._metadata, record);
|
|
4012
|
+
}
|
|
4013
|
+
}
|
|
4014
|
+
this._ownsMetadata = true;
|
|
4015
|
+
return this._metadata;
|
|
4016
|
+
}
|
|
2546
4017
|
/**
|
|
2547
4018
|
* Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
|
|
4019
|
+
*
|
|
4020
|
+
* The change applies to this frame only, never to a frame it was derived from or shares a header
|
|
4021
|
+
* with.
|
|
4022
|
+
*
|
|
2548
4023
|
* @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
|
|
2549
4024
|
*/
|
|
2550
4025
|
setFileMetadata(metadata) {
|
|
2551
|
-
|
|
2552
|
-
this._metadata = {
|
|
2553
|
-
QvBuildNo: 50667,
|
|
2554
|
-
CreatorDoc: "",
|
|
2555
|
-
CreateUtcTime: "",
|
|
2556
|
-
SourceCreateUtcTime: "",
|
|
2557
|
-
SourceFileUtcTime: "",
|
|
2558
|
-
SourceFileSize: -1,
|
|
2559
|
-
StaleUtcTime: "",
|
|
2560
|
-
TableName: "",
|
|
2561
|
-
Fields: {
|
|
2562
|
-
QvdFieldHeader: this._columns.map((column) => ({
|
|
2563
|
-
FieldName: column,
|
|
2564
|
-
BitOffset: 0,
|
|
2565
|
-
BitWidth: 0,
|
|
2566
|
-
Bias: 0,
|
|
2567
|
-
NoOfSymbols: 0,
|
|
2568
|
-
Offset: 0,
|
|
2569
|
-
Length: 0,
|
|
2570
|
-
Comment: "",
|
|
2571
|
-
NumberFormat: {
|
|
2572
|
-
Type: "UNKNOWN",
|
|
2573
|
-
nDec: "0",
|
|
2574
|
-
UseThou: "0",
|
|
2575
|
-
Fmt: "",
|
|
2576
|
-
Dec: "",
|
|
2577
|
-
Thou: ""
|
|
2578
|
-
},
|
|
2579
|
-
Tags: {}
|
|
2580
|
-
}))
|
|
2581
|
-
},
|
|
2582
|
-
NoOfRecords: 0,
|
|
2583
|
-
RecordByteSize: 0,
|
|
2584
|
-
Offset: 0,
|
|
2585
|
-
Length: 0,
|
|
2586
|
-
Compression: "",
|
|
2587
|
-
Comment: "",
|
|
2588
|
-
EncryptionInfo: "",
|
|
2589
|
-
TableTags: "",
|
|
2590
|
-
ProfilingData: "",
|
|
2591
|
-
Lineage: {}
|
|
2592
|
-
};
|
|
2593
|
-
}
|
|
4026
|
+
const header = this._ownMetadata();
|
|
2594
4027
|
const modifiableFields = [
|
|
2595
4028
|
"qvBuildNo",
|
|
2596
4029
|
"creatorDoc",
|
|
@@ -2625,7 +4058,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2625
4058
|
};
|
|
2626
4059
|
modifiableFields.forEach((field) => {
|
|
2627
4060
|
if (metadata[field] !== void 0) {
|
|
2628
|
-
|
|
4061
|
+
header[fieldMapping[field]] = metadata[field];
|
|
2629
4062
|
}
|
|
2630
4063
|
});
|
|
2631
4064
|
}
|
|
@@ -2638,30 +4071,44 @@ var init_QvdDataFrame = __esm({
|
|
|
2638
4071
|
/**
|
|
2639
4072
|
* Sets modifiable field-level metadata for a specific field.
|
|
2640
4073
|
* Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
|
|
4074
|
+
*
|
|
4075
|
+
* Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
|
|
4076
|
+
* header does not describe. The change applies to this frame only, never to a frame it was derived
|
|
4077
|
+
* from or shares a header with.
|
|
4078
|
+
*
|
|
2641
4079
|
* @param {string} fieldName The name of the field.
|
|
2642
4080
|
* @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
|
|
4081
|
+
* @throws {QvdValidationError} If the frame has no column of that name.
|
|
2643
4082
|
*/
|
|
2644
4083
|
setFieldMetadata(fieldName, metadata) {
|
|
2645
|
-
if (!this.
|
|
2646
|
-
|
|
4084
|
+
if (!this._columns.includes(fieldName)) {
|
|
4085
|
+
throw new exports.QvdValidationError(`Column '${fieldName}' does not exist`, {
|
|
4086
|
+
column: fieldName,
|
|
4087
|
+
availableColumns: this._columns
|
|
4088
|
+
});
|
|
2647
4089
|
}
|
|
2648
|
-
|
|
4090
|
+
const header = this._ownMetadata();
|
|
4091
|
+
if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
|
|
4092
|
+
header.Fields = { QvdFieldHeader: [] };
|
|
4093
|
+
}
|
|
4094
|
+
let fields = header.Fields.QvdFieldHeader;
|
|
2649
4095
|
if (!Array.isArray(fields)) {
|
|
2650
4096
|
fields = [fields];
|
|
2651
|
-
|
|
4097
|
+
header.Fields.QvdFieldHeader = fields;
|
|
2652
4098
|
}
|
|
2653
|
-
|
|
2654
|
-
if (
|
|
2655
|
-
|
|
4099
|
+
let field = fields.find((f) => f.FieldName === fieldName);
|
|
4100
|
+
if (!field) {
|
|
4101
|
+
field = defaultFieldHeader(fieldName);
|
|
4102
|
+
fields.push(field);
|
|
2656
4103
|
}
|
|
2657
4104
|
if (metadata.comment !== void 0) {
|
|
2658
|
-
|
|
4105
|
+
field.Comment = metadata.comment;
|
|
2659
4106
|
}
|
|
2660
4107
|
if (metadata.numberFormat !== void 0) {
|
|
2661
|
-
|
|
4108
|
+
field.NumberFormat = metadata.numberFormat;
|
|
2662
4109
|
}
|
|
2663
4110
|
if (metadata.tags !== void 0) {
|
|
2664
|
-
|
|
4111
|
+
field.Tags = metadata.tags;
|
|
2665
4112
|
}
|
|
2666
4113
|
}
|
|
2667
4114
|
/**
|
|
@@ -2678,7 +4125,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2678
4125
|
type: typeof n
|
|
2679
4126
|
});
|
|
2680
4127
|
}
|
|
2681
|
-
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
|
|
4128
|
+
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
|
|
2682
4129
|
}
|
|
2683
4130
|
/**
|
|
2684
4131
|
* Returns the last n rows of the data frame.
|
|
@@ -2694,7 +4141,13 @@ var init_QvdDataFrame = __esm({
|
|
|
2694
4141
|
type: typeof n
|
|
2695
4142
|
});
|
|
2696
4143
|
}
|
|
2697
|
-
return new _QvdDataFrame(
|
|
4144
|
+
return new _QvdDataFrame(
|
|
4145
|
+
n === 0 ? [] : this._data.slice(-n),
|
|
4146
|
+
this._columns,
|
|
4147
|
+
this._metadata,
|
|
4148
|
+
null,
|
|
4149
|
+
this._storedSymbols
|
|
4150
|
+
);
|
|
2698
4151
|
}
|
|
2699
4152
|
/**
|
|
2700
4153
|
* Returns the selected rows of the data frame.
|
|
@@ -2722,7 +4175,9 @@ var init_QvdDataFrame = __esm({
|
|
|
2722
4175
|
return new _QvdDataFrame(
|
|
2723
4176
|
args.map((index) => this._data[index]),
|
|
2724
4177
|
this._columns,
|
|
2725
|
-
this._metadata
|
|
4178
|
+
this._metadata,
|
|
4179
|
+
null,
|
|
4180
|
+
this._storedSymbols
|
|
2726
4181
|
);
|
|
2727
4182
|
}
|
|
2728
4183
|
/**
|
|
@@ -2734,6 +4189,50 @@ var init_QvdDataFrame = __esm({
|
|
|
2734
4189
|
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
2735
4190
|
*/
|
|
2736
4191
|
at(row, column) {
|
|
4192
|
+
const index = this._cellIndex(row, column);
|
|
4193
|
+
return this._data[row][index];
|
|
4194
|
+
}
|
|
4195
|
+
/**
|
|
4196
|
+
* Returns the text of the value at the specified row and column.
|
|
4197
|
+
*
|
|
4198
|
+
* The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
|
|
4199
|
+
* `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
|
|
4200
|
+
* or the string it was spelled as - and is null for a number that was stored as a pure number.
|
|
4201
|
+
*
|
|
4202
|
+
* ```js
|
|
4203
|
+
* const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
|
|
4204
|
+
* df.at(0, 'date'); // -52593
|
|
4205
|
+
* df.textAt(0, 'date'); // '1756-01-01'
|
|
4206
|
+
* ```
|
|
4207
|
+
*
|
|
4208
|
+
* @param {number} row The index of the row.
|
|
4209
|
+
* @param {string} column The name of the column.
|
|
4210
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
4211
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4212
|
+
*/
|
|
4213
|
+
textAt(row, column) {
|
|
4214
|
+
const index = this._cellIndex(row, column);
|
|
4215
|
+
const value = this._data[row][index];
|
|
4216
|
+
if (typeof value === "string") {
|
|
4217
|
+
return value;
|
|
4218
|
+
}
|
|
4219
|
+
if (typeof value === "number") {
|
|
4220
|
+
const entry = storedSymbolsEntry(this._storedSymbols, column);
|
|
4221
|
+
return entry === null ? null : storedTextOf(entry, value);
|
|
4222
|
+
}
|
|
4223
|
+
const dual = asDual(value);
|
|
4224
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
4225
|
+
}
|
|
4226
|
+
/**
|
|
4227
|
+
* Checks a row and a column name, and returns the column's position.
|
|
4228
|
+
*
|
|
4229
|
+
* @param {number} row The index of the row.
|
|
4230
|
+
* @param {string} column The name of the column.
|
|
4231
|
+
* @return {number} The column's position.
|
|
4232
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4233
|
+
* @private
|
|
4234
|
+
*/
|
|
4235
|
+
_cellIndex(row, column) {
|
|
2737
4236
|
if (typeof row !== "number" || !Number.isInteger(row)) {
|
|
2738
4237
|
throw new exports.QvdValidationError("Row index must be an integer", {
|
|
2739
4238
|
provided: row,
|
|
@@ -2753,7 +4252,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2753
4252
|
availableColumns: this._columns
|
|
2754
4253
|
});
|
|
2755
4254
|
}
|
|
2756
|
-
return this.
|
|
4255
|
+
return this._columns.indexOf(column);
|
|
2757
4256
|
}
|
|
2758
4257
|
/**
|
|
2759
4258
|
* Selects the specified columns from the data frame.
|
|
@@ -2774,15 +4273,34 @@ var init_QvdDataFrame = __esm({
|
|
|
2774
4273
|
const indices = args.map((arg) => this._columns.indexOf(arg));
|
|
2775
4274
|
const data = this._data.map((row) => indices.map((index) => row[index]));
|
|
2776
4275
|
const columns = indices.map((index) => this._columns[index]);
|
|
2777
|
-
return new _QvdDataFrame(data, columns, this._metadata);
|
|
4276
|
+
return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
|
|
2778
4277
|
}
|
|
2779
4278
|
/**
|
|
2780
4279
|
* Returns the data frame as a dictionary.
|
|
2781
4280
|
*
|
|
2782
|
-
*
|
|
4281
|
+
* Everything a frame needs to write the same file again, as plain data: the header, and the
|
|
4282
|
+
* stored-symbol record that says what cells showing one half of a symbol were read from. So
|
|
4283
|
+
* `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
|
|
4284
|
+
* `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
|
|
4285
|
+
*
|
|
4286
|
+
* @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
|
|
2783
4287
|
*/
|
|
2784
4288
|
async toDict() {
|
|
2785
|
-
return
|
|
4289
|
+
return this.toJSON();
|
|
4290
|
+
}
|
|
4291
|
+
/**
|
|
4292
|
+
* The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
|
|
4293
|
+
* `fromDict(JSON.parse(...))` can revive.
|
|
4294
|
+
*
|
|
4295
|
+
* @return {QvdDataFrameDict} The data frame as a dictionary.
|
|
4296
|
+
*/
|
|
4297
|
+
toJSON() {
|
|
4298
|
+
return {
|
|
4299
|
+
columns: this._columns,
|
|
4300
|
+
data: this._data,
|
|
4301
|
+
metadata: this._metadata,
|
|
4302
|
+
storedSymbols: this._storedSymbols
|
|
4303
|
+
};
|
|
2786
4304
|
}
|
|
2787
4305
|
/**
|
|
2788
4306
|
* Persists the data frame to a QVD file.
|
|
@@ -2811,6 +4329,33 @@ var init_QvdDataFrame = __esm({
|
|
|
2811
4329
|
* @param {Object} [options] Optional loading options.
|
|
2812
4330
|
* @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
|
|
2813
4331
|
* integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
|
|
4332
|
+
* This is the older name for `limit`; the two are the same option and passing both throws.
|
|
4333
|
+
* @param {number|null} [options.limit] Rows to read, counting from `offset`. The same number as
|
|
4334
|
+
* `maxRows`, spelled so that it reads correctly beside an offset.
|
|
4335
|
+
* @param {number} [options.offset=0] File row to start at. An offset past the end of the file
|
|
4336
|
+
* returns no rows rather than throwing, so a paging loop terminates on its own.
|
|
4337
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
4338
|
+
* appear in the result. Unselected fields have their symbols skipped entirely rather than
|
|
4339
|
+
* parsed and discarded. An unknown or repeated name throws.
|
|
4340
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
4341
|
+
* text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
|
|
4342
|
+
* `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
|
|
4343
|
+
* `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
|
|
4344
|
+
* implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
|
|
4345
|
+
* `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
|
|
4346
|
+
* double, a string and NULL read the same in every mode: a number, a string and null. Anything else
|
|
4347
|
+
* throws.
|
|
4348
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4349
|
+
* reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
|
|
4350
|
+
* reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
|
|
4351
|
+
* `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
|
|
4352
|
+
* `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
|
|
4353
|
+
* original string or dual back, and a value that two stored values read as is refused there
|
|
4354
|
+
* rather than written as either. Anything but a boolean throws.
|
|
4355
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
4356
|
+
* read proceeds - the same shape `toQvd`'s callback receives.
|
|
4357
|
+
* @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
|
|
4358
|
+
* which is a `DOMException` named `AbortError` unless you aborted with a reason of your own.
|
|
2814
4359
|
* @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
|
|
2815
4360
|
* must be within this directory, with symlinks resolved first, so a link inside it that points
|
|
2816
4361
|
* outside it is rejected. Defaults to the current working directory. To permit an entire
|
|
@@ -2823,17 +4368,64 @@ var init_QvdDataFrame = __esm({
|
|
|
2823
4368
|
* **Zero disables the memory check entirely.**
|
|
2824
4369
|
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
|
|
2825
4370
|
* a lazy load switches to the two-pass filtering path. Defaults to 50MB.
|
|
2826
|
-
* @throws {QvdValidationError} If
|
|
4371
|
+
* @throws {QvdValidationError} If a window option is not a non-negative integer, if both
|
|
4372
|
+
* `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
|
|
4373
|
+
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
2827
4374
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
2828
4375
|
*/
|
|
2829
4376
|
static async fromQvd(path3, options = {}) {
|
|
2830
4377
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
4378
|
+
return await new QvdFileReader2(path3, readerOptionsFrom(options)).load(windowFrom(options));
|
|
4379
|
+
}
|
|
4380
|
+
/**
|
|
4381
|
+
* Reads a QVD file in chunks, as an async generator of data frames.
|
|
4382
|
+
*
|
|
4383
|
+
* The file is opened, read and parsed once; only the index decode and the row building happen
|
|
4384
|
+
* per chunk, so what this bounds is row materialisation - the part that actually dominates a
|
|
4385
|
+
* large read's heap. It is **not** constant-memory reading of an arbitrarily large file: the
|
|
4386
|
+
* symbol table is parsed in full whatever the chunk size, because a stored index in the last
|
|
4387
|
+
* chunk can address the first symbol. On a high-cardinality file that table is the bulk of the
|
|
4388
|
+
* cost, and `readMetadata` is the only read that avoids it.
|
|
4389
|
+
*
|
|
4390
|
+
* ```js
|
|
4391
|
+
* for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
|
|
4392
|
+
* process(chunk.data);
|
|
4393
|
+
* }
|
|
4394
|
+
* ```
|
|
4395
|
+
*
|
|
4396
|
+
* A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
|
|
4397
|
+
* run its body.
|
|
4398
|
+
*
|
|
4399
|
+
* Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
|
|
4400
|
+
* them, so a chunk written on its own writes the symbols its cells were read from.
|
|
4401
|
+
*
|
|
4402
|
+
* @param {string} path The path to the QVD file.
|
|
4403
|
+
* @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
|
|
4404
|
+
* @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
|
|
4405
|
+
* @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
|
|
4406
|
+
* @param {number|null} [options.limit] Rows to cover, counting from `offset`.
|
|
4407
|
+
* @param {number} [options.offset=0] File row to start at.
|
|
4408
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
|
|
4409
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
|
|
4410
|
+
* its text, or a frozen `QvdDual` holding both. Anything else throws.
|
|
4411
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4412
|
+
* reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
4413
|
+
* `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
|
|
4414
|
+
* `storedSymbols`. Anything but a boolean throws.
|
|
4415
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
|
|
4416
|
+
* over the rows counts the whole window, not each chunk.
|
|
4417
|
+
* @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
|
|
4418
|
+
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
4419
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
|
|
4420
|
+
* charged for two chunks of rows rather than the window. Zero disables the check.
|
|
4421
|
+
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
|
|
4422
|
+
* windowed read switches to two-pass filtering.
|
|
4423
|
+
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
4424
|
+
*/
|
|
4425
|
+
static async *iterate(path3, options = {}) {
|
|
4426
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4427
|
+
const reader = new QvdFileReader2(path3, readerOptionsFrom(options));
|
|
4428
|
+
yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
|
|
2837
4429
|
}
|
|
2838
4430
|
/**
|
|
2839
4431
|
* Reads a QVD file's schema and header metadata, without reading its data.
|
|
@@ -2857,17 +4449,28 @@ var init_QvdDataFrame = __esm({
|
|
|
2857
4449
|
* @param {Object} [options] Optional reading options.
|
|
2858
4450
|
* @param {string} [options.allowedDir] Optional allowed directory path, applied exactly as it
|
|
2859
4451
|
* is for `fromQvd`.
|
|
4452
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`, as on
|
|
4453
|
+
* the reads that return data. Only the `read` and `header` stages occur here; there are no
|
|
4454
|
+
* symbols to parse and no rows to build.
|
|
4455
|
+
* @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
|
|
2860
4456
|
* @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
|
|
2861
4457
|
*/
|
|
2862
4458
|
static async readMetadata(path3, options = {}) {
|
|
2863
4459
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2864
|
-
return await new QvdFileReader2(path3,
|
|
4460
|
+
return await new QvdFileReader2(path3, metadataOptionsFrom(options)).loadMetadata();
|
|
2865
4461
|
}
|
|
2866
4462
|
/**
|
|
2867
4463
|
* Constructs a data frame from a dictionary.
|
|
2868
4464
|
*
|
|
2869
|
-
*
|
|
4465
|
+
* Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
|
|
4466
|
+
* rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
|
|
4467
|
+
* spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
|
|
4468
|
+
* here rather than written.
|
|
4469
|
+
*
|
|
4470
|
+
* @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
|
|
2870
4471
|
* @return {Promise<QvdDataFrame>} The constructed data frame.
|
|
4472
|
+
* @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
|
|
4473
|
+
* or `storedSymbols` is malformed or names a field that is not one of the columns.
|
|
2871
4474
|
*/
|
|
2872
4475
|
static async fromDict(data) {
|
|
2873
4476
|
if (!data.columns) {
|
|
@@ -2880,18 +4483,292 @@ var init_QvdDataFrame = __esm({
|
|
|
2880
4483
|
data
|
|
2881
4484
|
});
|
|
2882
4485
|
}
|
|
2883
|
-
|
|
4486
|
+
const { metadata = null, storedSymbols = null } = data;
|
|
4487
|
+
if (metadata !== null && !isPlainObject(metadata)) {
|
|
4488
|
+
throw new exports.QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
|
|
4489
|
+
type: typeof metadata
|
|
4490
|
+
});
|
|
4491
|
+
}
|
|
4492
|
+
const record = normaliseStoredSymbols(storedSymbols);
|
|
4493
|
+
if (record !== null) {
|
|
4494
|
+
const unknown = record.find((entry) => !data.columns.includes(entry.field));
|
|
4495
|
+
if (unknown !== void 0) {
|
|
4496
|
+
throw new exports.QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
|
|
4497
|
+
field: unknown.field,
|
|
4498
|
+
availableColumns: data.columns
|
|
4499
|
+
});
|
|
4500
|
+
}
|
|
4501
|
+
}
|
|
4502
|
+
return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
|
|
2884
4503
|
}
|
|
2885
4504
|
};
|
|
2886
4505
|
}
|
|
2887
4506
|
});
|
|
2888
4507
|
|
|
4508
|
+
// src/QvdSymbol.js
|
|
4509
|
+
init_QvdErrors();
|
|
4510
|
+
init_cellRules();
|
|
4511
|
+
init_symbolBytes();
|
|
4512
|
+
function checkInteger(value, context) {
|
|
4513
|
+
if (typeof value === "number" && isStoredAsInt(value)) {
|
|
4514
|
+
return;
|
|
4515
|
+
}
|
|
4516
|
+
throw new exports.QvdValidationError(
|
|
4517
|
+
`The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
|
|
4518
|
+
{ ...context, half: "integer", type: typeof value }
|
|
4519
|
+
);
|
|
4520
|
+
}
|
|
4521
|
+
__name(checkInteger, "checkInteger");
|
|
4522
|
+
var QvdSymbol = class _QvdSymbol {
|
|
4523
|
+
static {
|
|
4524
|
+
__name(this, "QvdSymbol");
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Constructs a new QVD symbol.
|
|
4528
|
+
*
|
|
4529
|
+
* @param {number|null} intValue The integer value.
|
|
4530
|
+
* @param {number|null} doubleValue The double value.
|
|
4531
|
+
* @param {string|null} stringValue The string value.
|
|
4532
|
+
*/
|
|
4533
|
+
constructor(intValue, doubleValue, stringValue) {
|
|
4534
|
+
this._intValue = intValue;
|
|
4535
|
+
this._doubleValue = doubleValue;
|
|
4536
|
+
this._stringValue = stringValue;
|
|
4537
|
+
}
|
|
4538
|
+
/**
|
|
4539
|
+
* Returns the integer value of this symbol.
|
|
4540
|
+
*
|
|
4541
|
+
* @return {number|null} The integer value.
|
|
4542
|
+
*/
|
|
4543
|
+
get intValue() {
|
|
4544
|
+
return this._intValue;
|
|
4545
|
+
}
|
|
4546
|
+
/**
|
|
4547
|
+
* Returns the double value of this symbol.
|
|
4548
|
+
*
|
|
4549
|
+
* @return {number|null} The double value.
|
|
4550
|
+
*/
|
|
4551
|
+
get doubleValue() {
|
|
4552
|
+
return this._doubleValue;
|
|
4553
|
+
}
|
|
4554
|
+
/**
|
|
4555
|
+
* Returns the string value of this symbol.
|
|
4556
|
+
*
|
|
4557
|
+
* @return {string|null} The string value.
|
|
4558
|
+
*/
|
|
4559
|
+
get stringValue() {
|
|
4560
|
+
return this._stringValue;
|
|
4561
|
+
}
|
|
4562
|
+
/**
|
|
4563
|
+
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
4564
|
+
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
4565
|
+
*
|
|
4566
|
+
* @return {number|string|null} The primary value.
|
|
4567
|
+
*/
|
|
4568
|
+
toPrimaryValue() {
|
|
4569
|
+
if (null != this._stringValue) {
|
|
4570
|
+
return this._stringValue;
|
|
4571
|
+
} else if (null != this._intValue) {
|
|
4572
|
+
return this._intValue;
|
|
4573
|
+
} else if (null != this._doubleValue) {
|
|
4574
|
+
return this._doubleValue;
|
|
4575
|
+
} else {
|
|
4576
|
+
return null;
|
|
4577
|
+
}
|
|
4578
|
+
}
|
|
4579
|
+
/**
|
|
4580
|
+
* Converts the symbol to its byte representation.
|
|
4581
|
+
*
|
|
4582
|
+
* The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
|
|
4583
|
+
* double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
|
|
4584
|
+
* checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
|
|
4585
|
+
* outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
|
|
4586
|
+
* integer and a double silently lost the double, a text with a NUL produced a symbol that ends
|
|
4587
|
+
* early, and a text with an unpaired surrogate was written with U+FFFD in its place.
|
|
4588
|
+
*
|
|
4589
|
+
* A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
|
|
4590
|
+
* `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
|
|
4591
|
+
* `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
|
|
4592
|
+
*
|
|
4593
|
+
* @return {Buffer} The byte representation of the symbol.
|
|
4594
|
+
* @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
|
|
4595
|
+
* holds a half no symbol can store. The message names the half.
|
|
4596
|
+
*/
|
|
4597
|
+
toByteRepresentation() {
|
|
4598
|
+
const intValue = this._intValue ?? null;
|
|
4599
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4600
|
+
const stringValue = this._stringValue ?? null;
|
|
4601
|
+
if (intValue !== null && doubleValue !== null) {
|
|
4602
|
+
throw new exports.QvdValidationError("A symbol holds an integer or a double, not both", {
|
|
4603
|
+
intValue,
|
|
4604
|
+
doubleValue
|
|
4605
|
+
});
|
|
4606
|
+
}
|
|
4607
|
+
if (intValue === null && doubleValue === null && stringValue === null) {
|
|
4608
|
+
throw new exports.QvdValidationError("The symbol does not contain any value.", {
|
|
4609
|
+
intValue,
|
|
4610
|
+
doubleValue,
|
|
4611
|
+
stringValue
|
|
4612
|
+
});
|
|
4613
|
+
}
|
|
4614
|
+
if (intValue !== null) {
|
|
4615
|
+
checkInteger(intValue, {});
|
|
4616
|
+
}
|
|
4617
|
+
if (doubleValue !== null) {
|
|
4618
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4619
|
+
}
|
|
4620
|
+
if (stringValue !== null) {
|
|
4621
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4622
|
+
}
|
|
4623
|
+
const number = intValue ?? doubleValue;
|
|
4624
|
+
const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
|
|
4625
|
+
const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
|
|
4626
|
+
writeSymbol(buffer, 0, kind, number, stringValue);
|
|
4627
|
+
return buffer;
|
|
4628
|
+
}
|
|
4629
|
+
/**
|
|
4630
|
+
* Checks if this symbol is equal to another symbol.
|
|
4631
|
+
*
|
|
4632
|
+
* By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
|
|
4633
|
+
* `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
|
|
4634
|
+
* answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
|
|
4635
|
+
* or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
|
|
4636
|
+
* storage kind, so either kind matches.
|
|
4637
|
+
*
|
|
4638
|
+
* @param {*} value The object to compare with.
|
|
4639
|
+
* @return {boolean} True if the objects are equal, false otherwise.
|
|
4640
|
+
*/
|
|
4641
|
+
equals(value) {
|
|
4642
|
+
if (value === null || typeof value !== "object") {
|
|
4643
|
+
return false;
|
|
4644
|
+
}
|
|
4645
|
+
const intValue = this._intValue ?? null;
|
|
4646
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4647
|
+
const stringValue = this._stringValue ?? null;
|
|
4648
|
+
const dual = asDual(value);
|
|
4649
|
+
if (dual !== null) {
|
|
4650
|
+
return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
|
|
4651
|
+
}
|
|
4652
|
+
if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
|
|
4653
|
+
return false;
|
|
4654
|
+
}
|
|
4655
|
+
return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
|
|
4656
|
+
}
|
|
4657
|
+
/**
|
|
4658
|
+
* Constructs a pure integer value symbol.
|
|
4659
|
+
*
|
|
4660
|
+
* @param {number} intValue The integer value.
|
|
4661
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4662
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
|
|
4663
|
+
*/
|
|
4664
|
+
static fromIntValue(intValue) {
|
|
4665
|
+
checkInteger(intValue, {});
|
|
4666
|
+
return new _QvdSymbol(intValue, null, null);
|
|
4667
|
+
}
|
|
4668
|
+
/**
|
|
4669
|
+
* Constructs a pure double value symbol.
|
|
4670
|
+
*
|
|
4671
|
+
* @param {number} doubleValue The double value.
|
|
4672
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4673
|
+
* @throws {QvdValidationError} If the double is not a finite number.
|
|
4674
|
+
*/
|
|
4675
|
+
static fromDoubleValue(doubleValue) {
|
|
4676
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4677
|
+
return new _QvdSymbol(null, doubleValue, null);
|
|
4678
|
+
}
|
|
4679
|
+
/**
|
|
4680
|
+
* Constructs a pure string value symbol.
|
|
4681
|
+
*
|
|
4682
|
+
* @param {string} stringValue The string value.
|
|
4683
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4684
|
+
* @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
|
|
4685
|
+
*/
|
|
4686
|
+
static fromStringValue(stringValue) {
|
|
4687
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4688
|
+
return new _QvdSymbol(null, null, stringValue);
|
|
4689
|
+
}
|
|
4690
|
+
/**
|
|
4691
|
+
* Constructs a dual value symbol from an integer and a string value.
|
|
4692
|
+
*
|
|
4693
|
+
* @param {number} intValue The integer value.
|
|
4694
|
+
* @param {string} stringValue The string value.
|
|
4695
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4696
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
|
|
4697
|
+
* is not a string, or holds a NUL or an unpaired surrogate.
|
|
4698
|
+
*/
|
|
4699
|
+
static fromDualIntValue(intValue, stringValue) {
|
|
4700
|
+
checkInteger(intValue, {});
|
|
4701
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4702
|
+
return new _QvdSymbol(intValue, null, stringValue);
|
|
4703
|
+
}
|
|
4704
|
+
/**
|
|
4705
|
+
* Constructs a dual value symbol from a double and a string value.
|
|
4706
|
+
*
|
|
4707
|
+
* @param {number} doubleValue The double value.
|
|
4708
|
+
* @param {string} stringValue The string value.
|
|
4709
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4710
|
+
* @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
|
|
4711
|
+
* or holds a NUL or an unpaired surrogate.
|
|
4712
|
+
*/
|
|
4713
|
+
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
4714
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4715
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4716
|
+
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
4717
|
+
}
|
|
4718
|
+
};
|
|
4719
|
+
|
|
4720
|
+
// src/index.js
|
|
4721
|
+
init_QvdDual();
|
|
4722
|
+
|
|
4723
|
+
// src/util/qlikDate.js
|
|
4724
|
+
init_QvdErrors();
|
|
4725
|
+
init_cellRules();
|
|
4726
|
+
var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
|
|
4727
|
+
var MS_PER_DAY = 864e5;
|
|
4728
|
+
var MAX_DATE_MS = 864e13;
|
|
4729
|
+
var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4730
|
+
var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4731
|
+
function qlikSerialToDate(serial) {
|
|
4732
|
+
const dual = asDual(serial);
|
|
4733
|
+
const number = dual === null ? serial : dual.number;
|
|
4734
|
+
if (typeof number !== "number" || !Number.isFinite(number)) {
|
|
4735
|
+
throw new exports.QvdValidationError("A Qlik date serial must be a finite number", {
|
|
4736
|
+
provided: describeType(number),
|
|
4737
|
+
type: typeof serial
|
|
4738
|
+
});
|
|
4739
|
+
}
|
|
4740
|
+
const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
|
|
4741
|
+
if (!(Math.abs(ms) <= MAX_DATE_MS)) {
|
|
4742
|
+
throw new exports.QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
|
|
4743
|
+
serial: number,
|
|
4744
|
+
minSerial: MIN_SERIAL,
|
|
4745
|
+
maxSerial: MAX_SERIAL
|
|
4746
|
+
});
|
|
4747
|
+
}
|
|
4748
|
+
return new Date(ms);
|
|
4749
|
+
}
|
|
4750
|
+
__name(qlikSerialToDate, "qlikSerialToDate");
|
|
4751
|
+
function dateToQlikSerial(date) {
|
|
4752
|
+
const ms = util.types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
|
|
4753
|
+
if (Number.isNaN(ms)) {
|
|
4754
|
+
throw new exports.QvdValidationError("dateToQlikSerial needs a valid Date", {
|
|
4755
|
+
provided: describeType(date),
|
|
4756
|
+
type: typeof date
|
|
4757
|
+
});
|
|
4758
|
+
}
|
|
4759
|
+
return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4760
|
+
}
|
|
4761
|
+
__name(dateToQlikSerial, "dateToQlikSerial");
|
|
4762
|
+
|
|
2889
4763
|
// src/index.js
|
|
2890
|
-
init_QvdSymbol();
|
|
2891
4764
|
init_QvdDataFrame();
|
|
2892
4765
|
init_QvdColumnTable();
|
|
2893
4766
|
init_QvdFileReader();
|
|
2894
4767
|
init_QvdFileWriter();
|
|
2895
4768
|
init_QvdErrors();
|
|
4769
|
+
|
|
4770
|
+
exports.QvdSymbol = QvdSymbol;
|
|
4771
|
+
exports.dateToQlikSerial = dateToQlikSerial;
|
|
4772
|
+
exports.qlikSerialToDate = qlikSerialToDate;
|
|
2896
4773
|
//# sourceMappingURL=index.cjs.map
|
|
2897
4774
|
//# sourceMappingURL=index.cjs.map
|