qvdjs 0.10.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -1101
- package/dist/index.cjs +2612 -735
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2610 -737
- package/dist/index.js.map +1 -1
- package/img/logo/qvdjs_logo-512.png +0 -0
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -5,9 +5,11 @@ import xml2 from 'xml2js';
|
|
|
5
5
|
import assert2 from 'assert';
|
|
6
6
|
import os from 'os';
|
|
7
7
|
import v8 from 'v8';
|
|
8
|
+
import { types } from 'util';
|
|
8
9
|
|
|
9
10
|
var __defProp = Object.defineProperty;
|
|
10
11
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
12
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
11
13
|
var __esm = (fn, res) => function __init() {
|
|
12
14
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
13
15
|
};
|
|
@@ -17,10 +19,14 @@ var __export = (target, all) => {
|
|
|
17
19
|
};
|
|
18
20
|
|
|
19
21
|
// src/QvdErrors.js
|
|
20
|
-
var QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
|
|
22
|
+
var ERROR_NAMES, QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
|
|
21
23
|
var init_QvdErrors = __esm({
|
|
22
24
|
"src/QvdErrors.js"() {
|
|
25
|
+
ERROR_NAMES = /* @__PURE__ */ new Map();
|
|
23
26
|
QvdError = class extends Error {
|
|
27
|
+
static {
|
|
28
|
+
__name(this, "QvdError");
|
|
29
|
+
}
|
|
24
30
|
/**
|
|
25
31
|
* Constructs a new QVD error.
|
|
26
32
|
*
|
|
@@ -30,13 +36,16 @@ var init_QvdErrors = __esm({
|
|
|
30
36
|
*/
|
|
31
37
|
constructor(message, code, context = {}) {
|
|
32
38
|
super(message);
|
|
33
|
-
this.name =
|
|
39
|
+
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
34
40
|
this.code = code;
|
|
35
41
|
this.context = context;
|
|
36
42
|
Error.captureStackTrace(this, this.constructor);
|
|
37
43
|
}
|
|
38
44
|
};
|
|
39
45
|
QvdParseError = class extends QvdError {
|
|
46
|
+
static {
|
|
47
|
+
__name(this, "QvdParseError");
|
|
48
|
+
}
|
|
40
49
|
/**
|
|
41
50
|
* Constructs a new QVD parse error.
|
|
42
51
|
*
|
|
@@ -48,6 +57,9 @@ var init_QvdErrors = __esm({
|
|
|
48
57
|
}
|
|
49
58
|
};
|
|
50
59
|
QvdValidationError = class extends QvdError {
|
|
60
|
+
static {
|
|
61
|
+
__name(this, "QvdValidationError");
|
|
62
|
+
}
|
|
51
63
|
/**
|
|
52
64
|
* Constructs a new QVD validation error.
|
|
53
65
|
*
|
|
@@ -59,6 +71,9 @@ var init_QvdErrors = __esm({
|
|
|
59
71
|
}
|
|
60
72
|
};
|
|
61
73
|
QvdIOError = class extends QvdError {
|
|
74
|
+
static {
|
|
75
|
+
__name(this, "QvdIOError");
|
|
76
|
+
}
|
|
62
77
|
/**
|
|
63
78
|
* Constructs a new QVD IO error.
|
|
64
79
|
*
|
|
@@ -70,6 +85,9 @@ var init_QvdErrors = __esm({
|
|
|
70
85
|
}
|
|
71
86
|
};
|
|
72
87
|
QvdCorruptedError = class extends QvdError {
|
|
88
|
+
static {
|
|
89
|
+
__name(this, "QvdCorruptedError");
|
|
90
|
+
}
|
|
73
91
|
/**
|
|
74
92
|
* Constructs a new QVD corrupted error.
|
|
75
93
|
*
|
|
@@ -81,6 +99,9 @@ var init_QvdErrors = __esm({
|
|
|
81
99
|
}
|
|
82
100
|
};
|
|
83
101
|
QvdSecurityError = class extends QvdError {
|
|
102
|
+
static {
|
|
103
|
+
__name(this, "QvdSecurityError");
|
|
104
|
+
}
|
|
84
105
|
/**
|
|
85
106
|
* Constructs a new QVD security error.
|
|
86
107
|
*
|
|
@@ -91,163 +112,591 @@ var init_QvdErrors = __esm({
|
|
|
91
112
|
super(message, "QVD_SECURITY_ERROR", context);
|
|
92
113
|
}
|
|
93
114
|
};
|
|
115
|
+
ERROR_NAMES.set(QvdError, "QvdError");
|
|
116
|
+
ERROR_NAMES.set(QvdParseError, "QvdParseError");
|
|
117
|
+
ERROR_NAMES.set(QvdValidationError, "QvdValidationError");
|
|
118
|
+
ERROR_NAMES.set(QvdIOError, "QvdIOError");
|
|
119
|
+
ERROR_NAMES.set(QvdCorruptedError, "QvdCorruptedError");
|
|
120
|
+
ERROR_NAMES.set(QvdSecurityError, "QvdSecurityError");
|
|
94
121
|
}
|
|
95
122
|
});
|
|
96
123
|
|
|
97
|
-
// src/
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
124
|
+
// src/util/cellRules.js
|
|
125
|
+
function asDual(value) {
|
|
126
|
+
if (value === null || typeof value !== "object") {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
if (value[DUAL_BRAND] === true) {
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
if (!isPlainObject(value)) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const keys = Object.keys(value);
|
|
137
|
+
return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
|
|
138
|
+
} catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function isPlainObject(value) {
|
|
143
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
const prototype = Object.getPrototypeOf(value);
|
|
147
|
+
return prototype === null || Object.getPrototypeOf(prototype) === null;
|
|
148
|
+
}
|
|
149
|
+
function isNumericText(text) {
|
|
150
|
+
return text.trim() !== "" && Number.isFinite(Number(text));
|
|
151
|
+
}
|
|
152
|
+
function isStoredAsInt(value) {
|
|
153
|
+
return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
|
|
154
|
+
}
|
|
155
|
+
function numberProblem(value) {
|
|
156
|
+
if (typeof value !== "number") {
|
|
157
|
+
return `is ${describeType(value)}, not a number`;
|
|
158
|
+
}
|
|
159
|
+
return Number.isFinite(value) ? null : `is ${String(value)}`;
|
|
160
|
+
}
|
|
161
|
+
function textProblem(value) {
|
|
162
|
+
if (typeof value !== "string") {
|
|
163
|
+
return { reason: "type" };
|
|
164
|
+
}
|
|
165
|
+
const position = value.indexOf(NUL);
|
|
166
|
+
if (position !== -1) {
|
|
167
|
+
return { reason: "nul", position };
|
|
168
|
+
}
|
|
169
|
+
return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
|
|
170
|
+
}
|
|
171
|
+
function unpairedSurrogateIndex(value) {
|
|
172
|
+
for (let index = 0; index < value.length; index++) {
|
|
173
|
+
const unit = value.charCodeAt(index);
|
|
174
|
+
if (unit < 55296 || unit > 57343) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const next = value.charCodeAt(index + 1);
|
|
178
|
+
if (unit <= 56319 && next >= 56320 && next <= 57343) {
|
|
179
|
+
index++;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
return index;
|
|
183
|
+
}
|
|
184
|
+
return -1;
|
|
185
|
+
}
|
|
186
|
+
function checkNumber(value, subject, context) {
|
|
187
|
+
if (numberProblem(value) === null) {
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (subject === null && typeof value === "number") {
|
|
191
|
+
throw new QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
|
|
192
|
+
...context,
|
|
193
|
+
provided: String(value)
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
throw new QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
|
|
197
|
+
...context,
|
|
198
|
+
type: typeof value
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
function checkText(value, subject, context) {
|
|
202
|
+
const problem = textProblem(value);
|
|
203
|
+
if (problem === null) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (problem.reason === "type") {
|
|
207
|
+
throw new QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
|
|
208
|
+
...context,
|
|
209
|
+
type: typeof value
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (problem.reason === "surrogate") {
|
|
213
|
+
throw new QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
|
|
214
|
+
...context,
|
|
215
|
+
position: problem.position
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
throw new QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
|
|
219
|
+
}
|
|
220
|
+
function constructorName(value) {
|
|
221
|
+
try {
|
|
222
|
+
const name = value.constructor?.name;
|
|
223
|
+
return typeof name === "string" && name !== "" ? name : null;
|
|
224
|
+
} catch {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function describeType(value) {
|
|
229
|
+
if (value === null) {
|
|
230
|
+
return "null";
|
|
231
|
+
}
|
|
232
|
+
if (typeof value === "number") {
|
|
233
|
+
return Number.isFinite(value) ? "a number" : String(value);
|
|
234
|
+
}
|
|
235
|
+
if (typeof value === "undefined") {
|
|
236
|
+
return "undefined";
|
|
237
|
+
}
|
|
238
|
+
if (typeof value !== "object") {
|
|
239
|
+
return `a ${typeof value}`;
|
|
240
|
+
}
|
|
241
|
+
let name;
|
|
242
|
+
try {
|
|
243
|
+
if (Array.isArray(value)) {
|
|
244
|
+
return "an array";
|
|
245
|
+
}
|
|
246
|
+
name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
|
|
247
|
+
if (name === "Object") {
|
|
248
|
+
if (!isPlainObject(value)) {
|
|
249
|
+
return "an object whose prototype is not Object.prototype";
|
|
250
|
+
}
|
|
251
|
+
const keys = Object.keys(value);
|
|
252
|
+
if (keys.length === 0) {
|
|
253
|
+
return "a plain object";
|
|
254
|
+
}
|
|
255
|
+
return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
|
|
256
|
+
}
|
|
257
|
+
} catch {
|
|
258
|
+
return "an object";
|
|
259
|
+
}
|
|
260
|
+
return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
|
|
261
|
+
}
|
|
262
|
+
var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
|
|
263
|
+
var init_cellRules = __esm({
|
|
264
|
+
"src/util/cellRules.js"() {
|
|
101
265
|
init_QvdErrors();
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
266
|
+
INT32_MIN = -2147483648;
|
|
267
|
+
INT32_MAX = 2147483647;
|
|
268
|
+
NUL = String.fromCharCode(0);
|
|
269
|
+
DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
|
|
270
|
+
__name(asDual, "asDual");
|
|
271
|
+
__name(isPlainObject, "isPlainObject");
|
|
272
|
+
__name(isNumericText, "isNumericText");
|
|
273
|
+
__name(isStoredAsInt, "isStoredAsInt");
|
|
274
|
+
__name(numberProblem, "numberProblem");
|
|
275
|
+
__name(textProblem, "textProblem");
|
|
276
|
+
__name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
|
|
277
|
+
__name(checkNumber, "checkNumber");
|
|
278
|
+
__name(checkText, "checkText");
|
|
279
|
+
__name(constructorName, "constructorName");
|
|
280
|
+
__name(describeType, "describeType");
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
// src/util/symbolBytes.js
|
|
285
|
+
function kindOf(number, text) {
|
|
286
|
+
if (number === null) {
|
|
287
|
+
return 4;
|
|
288
|
+
}
|
|
289
|
+
if (isStoredAsInt(number)) {
|
|
290
|
+
return text === null ? 1 : 5;
|
|
291
|
+
}
|
|
292
|
+
return text === null ? 2 : 6;
|
|
293
|
+
}
|
|
294
|
+
function symbolByteLength(kind, number, text) {
|
|
295
|
+
const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
|
|
296
|
+
const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
|
|
297
|
+
return 1 + numberBytes + textBytes;
|
|
298
|
+
}
|
|
299
|
+
function writeSymbol(buffer, offset, kind, number, text) {
|
|
300
|
+
buffer[offset++] = kind;
|
|
301
|
+
if (kind === 1 || kind === 5) {
|
|
302
|
+
offset = buffer.writeInt32LE(number, offset);
|
|
303
|
+
} else if (kind === 2 || kind === 6) {
|
|
304
|
+
offset = buffer.writeDoubleLE(number, offset);
|
|
305
|
+
}
|
|
306
|
+
if (kind >= 4) {
|
|
307
|
+
offset += buffer.write(text, offset, "utf8");
|
|
308
|
+
buffer[offset++] = 0;
|
|
309
|
+
}
|
|
310
|
+
return offset;
|
|
311
|
+
}
|
|
312
|
+
var init_symbolBytes = __esm({
|
|
313
|
+
"src/util/symbolBytes.js"() {
|
|
314
|
+
init_cellRules();
|
|
315
|
+
__name(kindOf, "kindOf");
|
|
316
|
+
__name(symbolByteLength, "symbolByteLength");
|
|
317
|
+
__name(writeSymbol, "writeSymbol");
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// src/QvdDual.js
|
|
322
|
+
function defineHalves(target, number, text) {
|
|
323
|
+
Object.defineProperties(target, {
|
|
324
|
+
number: { value: number, enumerable: true },
|
|
325
|
+
text: { value: text, enumerable: true }
|
|
326
|
+
});
|
|
327
|
+
Object.freeze(target);
|
|
328
|
+
}
|
|
329
|
+
function dualFromSymbol(number, text) {
|
|
330
|
+
const dual = Object.create(QvdDual.prototype);
|
|
331
|
+
defineHalves(dual, number, text);
|
|
332
|
+
return dual;
|
|
333
|
+
}
|
|
334
|
+
var QvdDual;
|
|
335
|
+
var init_QvdDual = __esm({
|
|
336
|
+
"src/QvdDual.js"() {
|
|
337
|
+
init_cellRules();
|
|
338
|
+
QvdDual = class {
|
|
339
|
+
static {
|
|
340
|
+
__name(this, "QvdDual");
|
|
114
341
|
}
|
|
115
342
|
/**
|
|
116
|
-
*
|
|
343
|
+
* Constructs a dual value.
|
|
344
|
+
*
|
|
345
|
+
* The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
|
|
346
|
+
* integer inside the int32 range is stored as a dual int, anything else as a dual double.
|
|
117
347
|
*
|
|
118
|
-
* @
|
|
348
|
+
* @param {number} number The numeric half. Must be a finite number.
|
|
349
|
+
* @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
|
|
350
|
+
* @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
|
|
119
351
|
*/
|
|
120
|
-
|
|
121
|
-
|
|
352
|
+
constructor(number, text) {
|
|
353
|
+
checkNumber(number, "The number of a dual value", { half: "number" });
|
|
354
|
+
checkText(text, "The text of a dual value", { half: "text" });
|
|
355
|
+
defineHalves(this, number, text);
|
|
122
356
|
}
|
|
123
357
|
/**
|
|
124
|
-
*
|
|
358
|
+
* Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
|
|
125
359
|
*
|
|
126
|
-
*
|
|
360
|
+
* The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
|
|
361
|
+
* conversion ran, not which half the caller meant, and the fix is the same for all three.
|
|
362
|
+
*
|
|
363
|
+
* @return {never}
|
|
364
|
+
* @throws {TypeError} Always.
|
|
127
365
|
*/
|
|
128
|
-
|
|
129
|
-
|
|
366
|
+
[Symbol.toPrimitive]() {
|
|
367
|
+
throw new TypeError(
|
|
368
|
+
`A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
|
|
369
|
+
);
|
|
130
370
|
}
|
|
131
371
|
/**
|
|
132
|
-
*
|
|
372
|
+
* Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
|
|
133
373
|
*
|
|
134
|
-
* @return {string
|
|
374
|
+
* @return {{number: number, text: string}} The dual as a plain object.
|
|
135
375
|
*/
|
|
136
|
-
|
|
137
|
-
return this.
|
|
376
|
+
toJSON() {
|
|
377
|
+
return { number: this.number, text: this.text };
|
|
138
378
|
}
|
|
139
379
|
/**
|
|
140
|
-
*
|
|
141
|
-
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
380
|
+
* How Node's `util.inspect` and `console.log` show a dual.
|
|
142
381
|
*
|
|
143
|
-
* @return {
|
|
382
|
+
* @return {string} For example `QvdDual(4.5, "4.50")`.
|
|
144
383
|
*/
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
return this._stringValue;
|
|
148
|
-
} else if (null != this._intValue) {
|
|
149
|
-
return this._intValue;
|
|
150
|
-
} else if (null != this._doubleValue) {
|
|
151
|
-
return this._doubleValue;
|
|
152
|
-
} else {
|
|
153
|
-
return null;
|
|
154
|
-
}
|
|
384
|
+
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
385
|
+
return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
|
|
155
386
|
}
|
|
156
|
-
/**
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
* @return {Buffer} The byte representation of the symbol.
|
|
160
|
-
*/
|
|
161
|
-
toByteRepresentation() {
|
|
162
|
-
if (this._intValue !== null && this._stringValue !== null) {
|
|
163
|
-
const intBuffer = Buffer.alloc(4);
|
|
164
|
-
intBuffer.writeInt32LE(this._intValue);
|
|
165
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
166
|
-
return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
|
|
167
|
-
} else if (this._doubleValue !== null && this._stringValue !== null) {
|
|
168
|
-
const floatBuffer = Buffer.alloc(8);
|
|
169
|
-
floatBuffer.writeDoubleLE(this._doubleValue);
|
|
170
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
171
|
-
return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
|
|
172
|
-
} else if (this._intValue !== null) {
|
|
173
|
-
const buffer = Buffer.alloc(4);
|
|
174
|
-
buffer.writeInt32LE(this._intValue);
|
|
175
|
-
return Buffer.concat([Buffer.from([1]), buffer]);
|
|
176
|
-
} else if (this._doubleValue !== null) {
|
|
177
|
-
const buffer = Buffer.alloc(8);
|
|
178
|
-
buffer.writeDoubleLE(this._doubleValue);
|
|
179
|
-
return Buffer.concat([Buffer.from([2]), buffer]);
|
|
180
|
-
} else if (this._stringValue !== null) {
|
|
181
|
-
const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
182
|
-
return Buffer.concat([Buffer.from([4]), buffer]);
|
|
183
|
-
} else {
|
|
184
|
-
throw new QvdValidationError("The symbol does not contain any value.", {
|
|
185
|
-
intValue: this._intValue,
|
|
186
|
-
doubleValue: this._doubleValue,
|
|
187
|
-
stringValue: this._stringValue
|
|
188
|
-
});
|
|
189
|
-
}
|
|
387
|
+
/** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
|
|
388
|
+
get [Symbol.toStringTag]() {
|
|
389
|
+
return "QvdDual";
|
|
190
390
|
}
|
|
191
391
|
/**
|
|
192
|
-
*
|
|
392
|
+
* Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
|
|
393
|
+
* own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
|
|
193
394
|
*
|
|
194
|
-
*
|
|
195
|
-
* @return {boolean} True if the objects are equal, false otherwise.
|
|
196
|
-
*/
|
|
197
|
-
equals(value) {
|
|
198
|
-
if (!(value instanceof _QvdSymbol)) {
|
|
199
|
-
return false;
|
|
200
|
-
}
|
|
201
|
-
return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Constructs a pure integer value symbol.
|
|
395
|
+
* Recognition only. The halves are checked when the value is written.
|
|
205
396
|
*
|
|
206
|
-
* @param {
|
|
207
|
-
* @return {
|
|
397
|
+
* @param {any} value Any value.
|
|
398
|
+
* @return {boolean} True for a dual cell.
|
|
208
399
|
*/
|
|
209
|
-
static
|
|
210
|
-
return
|
|
400
|
+
static isDual(value) {
|
|
401
|
+
return asDual(value) !== null;
|
|
211
402
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
403
|
+
};
|
|
404
|
+
Object.defineProperty(QvdDual.prototype, DUAL_BRAND, { value: true });
|
|
405
|
+
__name(defineHalves, "defineHalves");
|
|
406
|
+
__name(dualFromSymbol, "dualFromSymbol");
|
|
407
|
+
}
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
// src/util/readOptions.js
|
|
411
|
+
function requireRowCount(value, name, filePath) {
|
|
412
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
413
|
+
throw new QvdValidationError(`${name} must be a non-negative integer`, {
|
|
414
|
+
option: name,
|
|
415
|
+
provided: value,
|
|
416
|
+
type: typeof value,
|
|
417
|
+
file: filePath
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
return value;
|
|
421
|
+
}
|
|
422
|
+
function normaliseWindow(window, filePath) {
|
|
423
|
+
if (window === null || window === void 0) {
|
|
424
|
+
return { offset: 0, limit: null };
|
|
425
|
+
}
|
|
426
|
+
if (typeof window === "number") {
|
|
427
|
+
return { offset: 0, limit: requireRowCount(window, "maxRows", filePath) };
|
|
428
|
+
}
|
|
429
|
+
if (typeof window !== "object" || Array.isArray(window)) {
|
|
430
|
+
throw new QvdValidationError("The row window must be a number, null, or an {offset, limit} object", {
|
|
431
|
+
provided: window,
|
|
432
|
+
type: typeof window,
|
|
433
|
+
file: filePath
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
const { offset, limit, maxRows } = window;
|
|
437
|
+
const limitGiven = limit !== void 0 && limit !== null;
|
|
438
|
+
const maxRowsGiven = maxRows !== void 0 && maxRows !== null;
|
|
439
|
+
if (limitGiven && maxRowsGiven) {
|
|
440
|
+
throw new QvdValidationError("maxRows and limit are two names for the same option; pass one of them, not both", {
|
|
441
|
+
maxRows,
|
|
442
|
+
limit,
|
|
443
|
+
file: filePath
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
return {
|
|
447
|
+
offset: offset === void 0 || offset === null ? 0 : requireRowCount(offset, "offset", filePath),
|
|
448
|
+
limit: limitGiven ? requireRowCount(limit, "limit", filePath) : maxRowsGiven ? requireRowCount(maxRows, "maxRows", filePath) : null
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
function resolveWindow(window, totalRows) {
|
|
452
|
+
const rows = Number.isSafeInteger(totalRows) && totalRows > 0 ? totalRows : 0;
|
|
453
|
+
const offset = Math.min(window.offset, rows);
|
|
454
|
+
return {
|
|
455
|
+
offset,
|
|
456
|
+
limit: Math.max(0, Math.min(window.limit === null ? Infinity : window.limit, rows - offset))
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function selectFields(fields, requested, filePath) {
|
|
460
|
+
if (requested === null || requested === void 0) {
|
|
461
|
+
return fields;
|
|
462
|
+
}
|
|
463
|
+
if (!Array.isArray(requested)) {
|
|
464
|
+
throw new QvdValidationError("fields must be an array of field names", {
|
|
465
|
+
provided: requested,
|
|
466
|
+
type: typeof requested,
|
|
467
|
+
file: filePath
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
const available = fields.map((field) => field["FieldName"]);
|
|
471
|
+
if (requested.length === 0) {
|
|
472
|
+
throw new QvdValidationError("fields must name at least one field", {
|
|
473
|
+
availableColumns: available,
|
|
474
|
+
file: filePath
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
const seen = /* @__PURE__ */ new Set();
|
|
478
|
+
return requested.map((name) => {
|
|
479
|
+
if (typeof name !== "string") {
|
|
480
|
+
throw new QvdValidationError("Field names must be strings", {
|
|
481
|
+
provided: name,
|
|
482
|
+
type: typeof name,
|
|
483
|
+
availableColumns: available,
|
|
484
|
+
file: filePath
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
if (seen.has(name)) {
|
|
488
|
+
throw new QvdValidationError(`Field '${name}' is listed twice`, {
|
|
489
|
+
column: name,
|
|
490
|
+
fields: requested,
|
|
491
|
+
file: filePath
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
seen.add(name);
|
|
495
|
+
const index = available.indexOf(name);
|
|
496
|
+
if (index === -1) {
|
|
497
|
+
throw new QvdValidationError(`Column '${name}' does not exist`, {
|
|
498
|
+
column: name,
|
|
499
|
+
availableColumns: available,
|
|
500
|
+
file: filePath
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
return fields[index];
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
function normaliseDuals(value, filePath) {
|
|
507
|
+
if (value === void 0 || value === null) {
|
|
508
|
+
return "number";
|
|
509
|
+
}
|
|
510
|
+
if (!DUAL_MODES.includes(value)) {
|
|
511
|
+
throw new QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
512
|
+
option: "duals",
|
|
513
|
+
provided: value,
|
|
514
|
+
file: filePath
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
return value;
|
|
518
|
+
}
|
|
519
|
+
function normaliseCoerceNumericStrings(value, filePath) {
|
|
520
|
+
if (value === void 0 || value === null) {
|
|
521
|
+
return false;
|
|
522
|
+
}
|
|
523
|
+
if (typeof value !== "boolean") {
|
|
524
|
+
throw new QvdValidationError("coerceNumericStrings must be true or false", {
|
|
525
|
+
option: "coerceNumericStrings",
|
|
526
|
+
provided: value,
|
|
527
|
+
type: typeof value,
|
|
528
|
+
file: filePath
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
return value;
|
|
532
|
+
}
|
|
533
|
+
function readerOptionsFrom(options) {
|
|
534
|
+
return {
|
|
535
|
+
allowedDir: options.allowedDir,
|
|
536
|
+
memorySafetyFactor: options.memorySafetyFactor,
|
|
537
|
+
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
538
|
+
fields: options.fields === void 0 ? null : options.fields,
|
|
539
|
+
duals: options.duals,
|
|
540
|
+
coerceNumericStrings: options.coerceNumericStrings,
|
|
541
|
+
onProgress: options.onProgress,
|
|
542
|
+
signal: options.signal
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
function metadataOptionsFrom(options) {
|
|
546
|
+
return {
|
|
547
|
+
allowedDir: options.allowedDir,
|
|
548
|
+
onProgress: options.onProgress,
|
|
549
|
+
signal: options.signal
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
function windowFrom(options) {
|
|
553
|
+
return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
|
|
554
|
+
}
|
|
555
|
+
var DUAL_MODES;
|
|
556
|
+
var init_readOptions = __esm({
|
|
557
|
+
"src/util/readOptions.js"() {
|
|
558
|
+
init_QvdErrors();
|
|
559
|
+
__name(requireRowCount, "requireRowCount");
|
|
560
|
+
__name(normaliseWindow, "normaliseWindow");
|
|
561
|
+
__name(resolveWindow, "resolveWindow");
|
|
562
|
+
__name(selectFields, "selectFields");
|
|
563
|
+
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
564
|
+
__name(normaliseDuals, "normaliseDuals");
|
|
565
|
+
__name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
|
|
566
|
+
__name(readerOptionsFrom, "readerOptionsFrom");
|
|
567
|
+
__name(metadataOptionsFrom, "metadataOptionsFrom");
|
|
568
|
+
__name(windowFrom, "windowFrom");
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
// src/util/storedSymbols.js
|
|
573
|
+
function trustStoredSymbols(entries) {
|
|
574
|
+
const record = Object.freeze(entries);
|
|
575
|
+
trusted.add(record);
|
|
576
|
+
return record;
|
|
577
|
+
}
|
|
578
|
+
function attachStoredSymbols(metadata, record) {
|
|
579
|
+
if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
|
|
580
|
+
Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
function refuse(message, context) {
|
|
584
|
+
throw new QvdValidationError(message, context);
|
|
585
|
+
}
|
|
586
|
+
function normaliseStoredSymbols(record) {
|
|
587
|
+
if (record === null || record === void 0) {
|
|
588
|
+
return null;
|
|
589
|
+
}
|
|
590
|
+
if (typeof record === "object" && trusted.has(record)) {
|
|
591
|
+
return record;
|
|
592
|
+
}
|
|
593
|
+
if (!Array.isArray(record)) {
|
|
594
|
+
refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
|
|
595
|
+
option: "storedSymbols",
|
|
596
|
+
type: typeof record
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
const fields = /* @__PURE__ */ new Set();
|
|
600
|
+
const entries = record.map((entry, index) => {
|
|
601
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
|
|
602
|
+
refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
|
|
603
|
+
}
|
|
604
|
+
const { field, values, numbers, texts } = entry;
|
|
605
|
+
if (fields.has(field)) {
|
|
606
|
+
refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
|
|
607
|
+
}
|
|
608
|
+
fields.add(field);
|
|
609
|
+
if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
|
|
610
|
+
refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
|
|
611
|
+
field,
|
|
612
|
+
entry: index
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
for (let symbol = 0; symbol < values.length; symbol++) {
|
|
616
|
+
const value = values[symbol];
|
|
617
|
+
const number = numbers[symbol];
|
|
618
|
+
const text = texts[symbol];
|
|
619
|
+
const context = { field, symbol };
|
|
620
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
621
|
+
refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
|
|
622
|
+
...context,
|
|
623
|
+
type: typeof value
|
|
624
|
+
});
|
|
220
625
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
*
|
|
224
|
-
* @param {string} stringValue The string value.
|
|
225
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
226
|
-
*/
|
|
227
|
-
static fromStringValue(stringValue) {
|
|
228
|
-
return new _QvdSymbol(null, null, stringValue);
|
|
626
|
+
if (number !== null) {
|
|
627
|
+
checkNumber(number, "A stored symbol's number", context);
|
|
229
628
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
*
|
|
233
|
-
* @param {number} intValue The integer value.
|
|
234
|
-
* @param {string} stringValue The string value.
|
|
235
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
236
|
-
*/
|
|
237
|
-
static fromDualIntValue(intValue, stringValue) {
|
|
238
|
-
return new _QvdSymbol(intValue, null, stringValue);
|
|
629
|
+
if (text !== null) {
|
|
630
|
+
checkText(text, "A stored symbol's text", context);
|
|
239
631
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
*/
|
|
247
|
-
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
248
|
-
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
632
|
+
const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
|
|
633
|
+
if (!consistent) {
|
|
634
|
+
refuse(
|
|
635
|
+
number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
|
|
636
|
+
{ ...context, value, number, text }
|
|
637
|
+
);
|
|
249
638
|
}
|
|
250
|
-
}
|
|
639
|
+
}
|
|
640
|
+
return Object.freeze({
|
|
641
|
+
field,
|
|
642
|
+
values: Object.freeze(values.slice()),
|
|
643
|
+
numbers: Object.freeze(numbers.slice()),
|
|
644
|
+
texts: Object.freeze(texts.slice())
|
|
645
|
+
});
|
|
646
|
+
});
|
|
647
|
+
return trustStoredSymbols(entries);
|
|
648
|
+
}
|
|
649
|
+
function narrowStoredSymbols(record, columns) {
|
|
650
|
+
if (record === null) {
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
const kept = record.filter((entry) => columns.includes(entry.field));
|
|
654
|
+
return kept.length === record.length ? record : trustStoredSymbols(kept);
|
|
655
|
+
}
|
|
656
|
+
function storedSymbolsEntry(record, field) {
|
|
657
|
+
if (record === null) {
|
|
658
|
+
return null;
|
|
659
|
+
}
|
|
660
|
+
return record.find((entry) => entry.field === field) ?? null;
|
|
661
|
+
}
|
|
662
|
+
function firstTextByValue(entry) {
|
|
663
|
+
const byValue = /* @__PURE__ */ new Map();
|
|
664
|
+
for (let index = entry.values.length - 1; index >= 0; index--) {
|
|
665
|
+
const text = entry.texts[index];
|
|
666
|
+
if (text !== null) {
|
|
667
|
+
byValue.set(entry.values[index], text);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
return byValue;
|
|
671
|
+
}
|
|
672
|
+
function storedTextOf(entry, value) {
|
|
673
|
+
let byValue = firstTexts.get(entry);
|
|
674
|
+
if (byValue === void 0) {
|
|
675
|
+
byValue = firstTextByValue(entry);
|
|
676
|
+
firstTexts.set(entry, byValue);
|
|
677
|
+
}
|
|
678
|
+
return byValue.get(value) ?? null;
|
|
679
|
+
}
|
|
680
|
+
function sameValueZero(a, b) {
|
|
681
|
+
return a === b || a !== a && b !== b;
|
|
682
|
+
}
|
|
683
|
+
var STORED_SYMBOLS, trusted, firstTexts;
|
|
684
|
+
var init_storedSymbols = __esm({
|
|
685
|
+
"src/util/storedSymbols.js"() {
|
|
686
|
+
init_QvdErrors();
|
|
687
|
+
init_cellRules();
|
|
688
|
+
STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
|
|
689
|
+
trusted = /* @__PURE__ */ new WeakSet();
|
|
690
|
+
__name(trustStoredSymbols, "trustStoredSymbols");
|
|
691
|
+
__name(attachStoredSymbols, "attachStoredSymbols");
|
|
692
|
+
__name(refuse, "refuse");
|
|
693
|
+
__name(normaliseStoredSymbols, "normaliseStoredSymbols");
|
|
694
|
+
__name(narrowStoredSymbols, "narrowStoredSymbols");
|
|
695
|
+
__name(storedSymbolsEntry, "storedSymbolsEntry");
|
|
696
|
+
firstTexts = /* @__PURE__ */ new WeakMap();
|
|
697
|
+
__name(firstTextByValue, "firstTextByValue");
|
|
698
|
+
__name(storedTextOf, "storedTextOf");
|
|
699
|
+
__name(sameValueZero, "sameValueZero");
|
|
251
700
|
}
|
|
252
701
|
});
|
|
253
702
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
@@ -352,6 +801,10 @@ function validatePath(filePath, allowedDir) {
|
|
|
352
801
|
var init_validatePath = __esm({
|
|
353
802
|
"src/util/validatePath.js"() {
|
|
354
803
|
init_QvdErrors();
|
|
804
|
+
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
805
|
+
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
806
|
+
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
807
|
+
__name(validatePath, "validatePath");
|
|
355
808
|
}
|
|
356
809
|
});
|
|
357
810
|
|
|
@@ -401,6 +854,9 @@ var init_bitUtils = __esm({
|
|
|
401
854
|
"src/util/bitUtils.js"() {
|
|
402
855
|
MAX_BIT_WIDTH = 31;
|
|
403
856
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
857
|
+
__name(fieldGeometry, "fieldGeometry");
|
|
858
|
+
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
859
|
+
__name(writeBitField, "writeBitField");
|
|
404
860
|
}
|
|
405
861
|
});
|
|
406
862
|
|
|
@@ -409,14 +865,310 @@ var QvdFileWriter_exports = {};
|
|
|
409
865
|
__export(QvdFileWriter_exports, {
|
|
410
866
|
QvdFileWriter: () => QvdFileWriter
|
|
411
867
|
});
|
|
412
|
-
|
|
868
|
+
function notXmlIndex(value) {
|
|
869
|
+
for (let index = 0; index < value.length; index++) {
|
|
870
|
+
const unit = value.charCodeAt(index);
|
|
871
|
+
if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
|
|
872
|
+
return index;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
return -1;
|
|
876
|
+
}
|
|
877
|
+
function checkHeaderText(value, subject, context) {
|
|
878
|
+
checkText(value, subject, context);
|
|
879
|
+
const position = notXmlIndex(value);
|
|
880
|
+
if (position !== -1) {
|
|
881
|
+
const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
|
|
882
|
+
throw new QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
function checkHeaderTexts(value, property, owner, context) {
|
|
886
|
+
if (typeof value === "string") {
|
|
887
|
+
checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
|
|
888
|
+
} else if (Array.isArray(value)) {
|
|
889
|
+
value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
|
|
890
|
+
} else if (value !== null && typeof value === "object") {
|
|
891
|
+
for (const [key, item] of Object.entries(value)) {
|
|
892
|
+
checkHeaderTexts(item, `${property}.${key}`, owner, context);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
function validateColumnNames(columns, filePath) {
|
|
897
|
+
const seen = /* @__PURE__ */ new Set();
|
|
898
|
+
columns.forEach((name, index) => {
|
|
899
|
+
if (typeof name !== "string" || name.length === 0) {
|
|
900
|
+
throw new QvdValidationError("Field names must be non-empty strings", {
|
|
901
|
+
column: index,
|
|
902
|
+
provided: name,
|
|
903
|
+
type: typeof name,
|
|
904
|
+
file: filePath,
|
|
905
|
+
stage: "buildSymbolTable"
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
|
|
909
|
+
if (seen.has(name)) {
|
|
910
|
+
throw new QvdValidationError(`Field '${name}' appears twice`, {
|
|
911
|
+
column: name,
|
|
912
|
+
columnIndex: index,
|
|
913
|
+
file: filePath,
|
|
914
|
+
stage: "buildSymbolTable"
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
seen.add(name);
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
function refuseCell(value, column, row, filePath) {
|
|
921
|
+
let resemblesDual = false;
|
|
922
|
+
try {
|
|
923
|
+
resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
|
|
924
|
+
} catch {
|
|
925
|
+
}
|
|
926
|
+
throw new QvdValidationError(
|
|
927
|
+
`A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
|
|
928
|
+
{
|
|
929
|
+
column,
|
|
930
|
+
row,
|
|
931
|
+
type: typeof value,
|
|
932
|
+
constructor: constructorName(value) ?? void 0,
|
|
933
|
+
file: filePath,
|
|
934
|
+
stage: "buildSymbolTable"
|
|
935
|
+
}
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
function valuesAreNumbers(entry) {
|
|
939
|
+
const { values, numbers } = entry;
|
|
940
|
+
for (let index = 0; index < values.length; index++) {
|
|
941
|
+
if (!sameValueZero(values[index], numbers[index])) {
|
|
942
|
+
return false;
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
return true;
|
|
946
|
+
}
|
|
947
|
+
function newSlot(column, key, text) {
|
|
948
|
+
const slot = column.keys.length;
|
|
949
|
+
column.keys.push(key);
|
|
950
|
+
column.texts.push(text);
|
|
951
|
+
if (typeof key === "number") {
|
|
952
|
+
column.facts.hasNumber = true;
|
|
953
|
+
if (!Number.isInteger(key)) column.facts.hasFraction = true;
|
|
954
|
+
} else {
|
|
955
|
+
column.facts.hasNonNumber = true;
|
|
956
|
+
}
|
|
957
|
+
return slot;
|
|
958
|
+
}
|
|
959
|
+
function slotFor(column, key, text) {
|
|
960
|
+
let slot = column.byKey.get(key);
|
|
961
|
+
if (slot === void 0) {
|
|
962
|
+
slot = newSlot(column, key, text);
|
|
963
|
+
column.byKey.set(key, slot);
|
|
964
|
+
} else if (text !== null && column.texts[slot] === null && typeof key === "number") {
|
|
965
|
+
column.texts[slot] = text;
|
|
966
|
+
}
|
|
967
|
+
return slot;
|
|
968
|
+
}
|
|
969
|
+
function standsForSeveral(indices, numbers, texts) {
|
|
970
|
+
let number = null;
|
|
971
|
+
let string = null;
|
|
972
|
+
for (const index of indices) {
|
|
973
|
+
if (numbers[index] !== null) {
|
|
974
|
+
if (number === null) {
|
|
975
|
+
number = numbers[index];
|
|
976
|
+
} else if (!sameValueZero(number, numbers[index])) {
|
|
977
|
+
return true;
|
|
978
|
+
}
|
|
979
|
+
} else if (string === null) {
|
|
980
|
+
string = texts[index];
|
|
981
|
+
} else if (string !== texts[index]) {
|
|
982
|
+
return true;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
return number !== null && string !== null;
|
|
986
|
+
}
|
|
987
|
+
function ambiguousAsUncoercedText(entry) {
|
|
988
|
+
const { values, numbers, texts } = entry;
|
|
989
|
+
for (let index = 0; index < values.length; index++) {
|
|
990
|
+
if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
|
|
991
|
+
if (!isNumericText(texts[index])) {
|
|
992
|
+
return false;
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
const byShown = /* @__PURE__ */ new Map();
|
|
997
|
+
for (let index = 0; index < values.length; index++) {
|
|
998
|
+
const shown = texts[index] ?? numbers[index];
|
|
999
|
+
const indices = byShown.get(shown);
|
|
1000
|
+
if (indices === void 0) {
|
|
1001
|
+
byShown.set(shown, [index]);
|
|
1002
|
+
} else {
|
|
1003
|
+
indices.push(index);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
for (const indices of byShown.values()) {
|
|
1007
|
+
if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
|
|
1008
|
+
return true;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
return false;
|
|
1012
|
+
}
|
|
1013
|
+
function unambiguousWithoutDuals(column) {
|
|
1014
|
+
const { numbers, texts } = column.entry;
|
|
1015
|
+
for (const found of column.byValue.values()) {
|
|
1016
|
+
if (typeof found !== "number") {
|
|
1017
|
+
const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
|
|
1018
|
+
if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
|
|
1019
|
+
return false;
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
return true;
|
|
1024
|
+
}
|
|
1025
|
+
function ambiguityRemedy(column) {
|
|
1026
|
+
const { values, numbers, texts } = column.entry;
|
|
1027
|
+
let coerced = false;
|
|
1028
|
+
let other = false;
|
|
1029
|
+
for (const [value, found] of column.byValue) {
|
|
1030
|
+
if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
|
|
1031
|
+
if (typeof value === "number") {
|
|
1032
|
+
coerced = true;
|
|
1033
|
+
} else {
|
|
1034
|
+
other = true;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
if (!coerced) {
|
|
1039
|
+
return "Read the field with {duals: 'both'}, or write QvdDual cells.";
|
|
1040
|
+
}
|
|
1041
|
+
if (!other && !ambiguousAsUncoercedText(column.entry)) {
|
|
1042
|
+
return "Read the field without {coerceNumericStrings: true}.";
|
|
1043
|
+
}
|
|
1044
|
+
if (unambiguousWithoutDuals(column)) {
|
|
1045
|
+
return "Read the field with {duals: 'both'}.";
|
|
1046
|
+
}
|
|
1047
|
+
return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
|
|
1048
|
+
}
|
|
1049
|
+
function slotForCell(column, value, row, filePath) {
|
|
1050
|
+
if (column.byCell === column.byKey) {
|
|
1051
|
+
return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
|
|
1052
|
+
}
|
|
1053
|
+
const found = column.byValue.get(value);
|
|
1054
|
+
if (found === void 0) {
|
|
1055
|
+
return slotFor(column, value, null);
|
|
1056
|
+
}
|
|
1057
|
+
const { numbers, texts } = column.entry;
|
|
1058
|
+
const indices = typeof found === "number" ? [found] : found;
|
|
1059
|
+
if (standsForSeveral(indices, numbers, texts)) {
|
|
1060
|
+
const stored = [];
|
|
1061
|
+
for (const index of indices) {
|
|
1062
|
+
if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
|
|
1063
|
+
stored.push({ number: numbers[index], text: texts[index] });
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
throw new QvdValidationError(
|
|
1067
|
+
`The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
|
|
1068
|
+
{ column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
let number = null;
|
|
1072
|
+
let numberText = null;
|
|
1073
|
+
for (const index of indices) {
|
|
1074
|
+
if (numbers[index] === null) {
|
|
1075
|
+
return slotFor(column, texts[index], null);
|
|
1076
|
+
}
|
|
1077
|
+
number ??= numbers[index];
|
|
1078
|
+
numberText ??= texts[index];
|
|
1079
|
+
}
|
|
1080
|
+
return slotFor(column, number, numberText);
|
|
1081
|
+
}
|
|
1082
|
+
function slotForObject(column, value, row, filePath) {
|
|
1083
|
+
const dual = asDual(value);
|
|
1084
|
+
if (dual === null) {
|
|
1085
|
+
refuseCell(value, column.name, row, filePath);
|
|
1086
|
+
}
|
|
1087
|
+
const { number, text } = dual;
|
|
1088
|
+
if (numberProblem(number) !== null) {
|
|
1089
|
+
checkNumber(number, "The number of a dual value", {
|
|
1090
|
+
column: column.name,
|
|
1091
|
+
row,
|
|
1092
|
+
half: "number",
|
|
1093
|
+
file: filePath,
|
|
1094
|
+
stage: "buildSymbolTable"
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
if (textProblem(text) !== null) {
|
|
1098
|
+
checkText(text, "The text of a dual value", {
|
|
1099
|
+
column: column.name,
|
|
1100
|
+
row,
|
|
1101
|
+
half: "text",
|
|
1102
|
+
file: filePath,
|
|
1103
|
+
stage: "buildSymbolTable"
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
const slot = slotFor(column, number, text);
|
|
1107
|
+
if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
|
|
1108
|
+
column.byObject.set(value, slot);
|
|
1109
|
+
}
|
|
1110
|
+
return slot;
|
|
1111
|
+
}
|
|
1112
|
+
function pruneContradictedTags(tags, facts) {
|
|
1113
|
+
if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
|
|
1114
|
+
return tags || {};
|
|
1115
|
+
}
|
|
1116
|
+
const list = Array.isArray(tags.String) ? tags.String : [tags.String];
|
|
1117
|
+
const kept = list.filter((tag) => {
|
|
1118
|
+
if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
|
|
1119
|
+
if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
|
|
1120
|
+
if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
|
|
1121
|
+
return true;
|
|
1122
|
+
});
|
|
1123
|
+
if (kept.length === list.length) {
|
|
1124
|
+
return tags;
|
|
1125
|
+
}
|
|
1126
|
+
return kept.length === 0 ? {} : { ...tags, String: kept };
|
|
1127
|
+
}
|
|
1128
|
+
function resetContradictedNumberFormat(numberFormat, facts) {
|
|
1129
|
+
if (!numberFormat) {
|
|
1130
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1131
|
+
}
|
|
1132
|
+
if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
|
|
1133
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1134
|
+
}
|
|
1135
|
+
return numberFormat;
|
|
1136
|
+
}
|
|
1137
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, QvdFileWriter;
|
|
413
1138
|
var init_QvdFileWriter = __esm({
|
|
414
1139
|
"src/QvdFileWriter.js"() {
|
|
415
|
-
init_QvdSymbol();
|
|
416
1140
|
init_QvdErrors();
|
|
417
1141
|
init_validatePath();
|
|
418
1142
|
init_bitUtils();
|
|
419
|
-
|
|
1143
|
+
init_cellRules();
|
|
1144
|
+
init_symbolBytes();
|
|
1145
|
+
init_storedSymbols();
|
|
1146
|
+
__name(notXmlIndex, "notXmlIndex");
|
|
1147
|
+
__name(checkHeaderText, "checkHeaderText");
|
|
1148
|
+
__name(checkHeaderTexts, "checkHeaderTexts");
|
|
1149
|
+
__name(validateColumnNames, "validateColumnNames");
|
|
1150
|
+
__name(refuseCell, "refuseCell");
|
|
1151
|
+
OBJECT_MEMO_BASE = 64;
|
|
1152
|
+
__name(valuesAreNumbers, "valuesAreNumbers");
|
|
1153
|
+
__name(newSlot, "newSlot");
|
|
1154
|
+
__name(slotFor, "slotFor");
|
|
1155
|
+
__name(standsForSeveral, "standsForSeveral");
|
|
1156
|
+
__name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
|
|
1157
|
+
__name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
|
|
1158
|
+
__name(ambiguityRemedy, "ambiguityRemedy");
|
|
1159
|
+
__name(slotForCell, "slotForCell");
|
|
1160
|
+
__name(slotForObject, "slotForObject");
|
|
1161
|
+
NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
|
|
1162
|
+
TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
|
|
1163
|
+
WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
|
|
1164
|
+
NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
|
|
1165
|
+
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1166
|
+
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1167
|
+
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1168
|
+
QvdFileWriter = class {
|
|
1169
|
+
static {
|
|
1170
|
+
__name(this, "QvdFileWriter");
|
|
1171
|
+
}
|
|
420
1172
|
/**
|
|
421
1173
|
* Constructs a new QVD file writer.
|
|
422
1174
|
*
|
|
@@ -437,7 +1189,8 @@ var init_QvdFileWriter = __esm({
|
|
|
437
1189
|
this._onProgress = onProgress;
|
|
438
1190
|
this._header = null;
|
|
439
1191
|
this._symbolBuffer = null;
|
|
440
|
-
this.
|
|
1192
|
+
this._symbolCounts = null;
|
|
1193
|
+
this._symbolFacts = null;
|
|
441
1194
|
this._symbolTableMetadata = null;
|
|
442
1195
|
this._indexBuffer = null;
|
|
443
1196
|
this._symbolIndexByValue = null;
|
|
@@ -548,19 +1301,12 @@ var init_QvdFileWriter = __esm({
|
|
|
548
1301
|
BitOffset: this._indexTableMetadata?.[index][0],
|
|
549
1302
|
BitWidth: this._indexTableMetadata?.[index][1],
|
|
550
1303
|
Bias: this._indexTableMetadata?.[index][2],
|
|
551
|
-
NoOfSymbols: this.
|
|
1304
|
+
NoOfSymbols: this._symbolCounts?.[index],
|
|
552
1305
|
Offset: this._symbolTableMetadata?.[index][0],
|
|
553
1306
|
Length: this._symbolTableMetadata?.[index][1],
|
|
554
1307
|
Comment: existingField?.Comment || "",
|
|
555
|
-
NumberFormat: existingField?.NumberFormat
|
|
556
|
-
|
|
557
|
-
nDec: "0",
|
|
558
|
-
UseThou: "0",
|
|
559
|
-
Fmt: "",
|
|
560
|
-
Dec: "",
|
|
561
|
-
Thou: ""
|
|
562
|
-
},
|
|
563
|
-
Tags: existingField?.Tags || {}
|
|
1308
|
+
NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
|
|
1309
|
+
Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
|
|
564
1310
|
};
|
|
565
1311
|
})
|
|
566
1312
|
},
|
|
@@ -570,6 +1316,19 @@ var init_QvdFileWriter = __esm({
|
|
|
570
1316
|
Length: this._indexBuffer?.length
|
|
571
1317
|
}
|
|
572
1318
|
};
|
|
1319
|
+
const { Fields, ...table } = xmlObject.QvdTableHeader;
|
|
1320
|
+
for (const [property, value] of Object.entries(table)) {
|
|
1321
|
+
checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
|
|
1322
|
+
}
|
|
1323
|
+
for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
|
|
1324
|
+
for (const [property, value] of Object.entries(field)) {
|
|
1325
|
+
checkHeaderTexts(value, property, `field '${FieldName}'`, {
|
|
1326
|
+
column: FieldName,
|
|
1327
|
+
file: this._path,
|
|
1328
|
+
stage: "buildHeader"
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
573
1332
|
const builder = new xml2.Builder({
|
|
574
1333
|
renderOpts: {
|
|
575
1334
|
pretty: true,
|
|
@@ -583,25 +1342,49 @@ var init_QvdFileWriter = __esm({
|
|
|
583
1342
|
/**
|
|
584
1343
|
* Builds the symbol table of the QVD file.
|
|
585
1344
|
*
|
|
586
|
-
*
|
|
587
|
-
*
|
|
588
|
-
*
|
|
1345
|
+
* One pass over the rows finds each column's distinct values in the order they first appear,
|
|
1346
|
+
* which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
|
|
1347
|
+
* column encodes them: every symbol is sized first, then written into one buffer of exactly that
|
|
1348
|
+
* size.
|
|
1349
|
+
*
|
|
1350
|
+
* What each value is stored as:
|
|
1351
|
+
*
|
|
1352
|
+
* | Cell | Symbol |
|
|
1353
|
+
* | --- | --- |
|
|
1354
|
+
* | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
|
|
1355
|
+
* | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
|
|
1356
|
+
* | any other finite number | pure double, type 2 |
|
|
1357
|
+
* | a string | pure string, type 4 |
|
|
1358
|
+
* | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
|
|
1359
|
+
* | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
|
|
1360
|
+
*
|
|
1361
|
+
* A number is a pure number, with no text. It used to be written as a dual whose text was
|
|
1362
|
+
* `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
|
|
1363
|
+
* file Qlik wrote with pure numbers came back out of a read and a write with every one of them
|
|
1364
|
+
* turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
|
|
1365
|
+
* A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
|
|
1366
|
+
* UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
|
|
1367
|
+
* an unpaired surrogate has no UTF-8 encoding at all.
|
|
1368
|
+
*
|
|
1369
|
+
* A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
|
|
1370
|
+
* symbol with the first text in row order, and a plain number with the same number as a dual joins
|
|
1371
|
+
* the dual, in either order - so no text a caller supplied is lost to a plain number that came
|
|
1372
|
+
* first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
|
|
589
1373
|
*
|
|
590
|
-
*
|
|
591
|
-
*
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
* 4. Serialize symbols to binary format and update metadata
|
|
1374
|
+
* A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
|
|
1375
|
+
* string read as a number. Its record says what each such cell stands for, so the frame writes back
|
|
1376
|
+
* the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
|
|
1377
|
+
* rows the cells were moved to. A cell the record maps to more than one stored value is refused.
|
|
595
1378
|
*
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
* - Lower memory pressure (no intermediate arrays per column)
|
|
1379
|
+
* The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
|
|
1380
|
+
* for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
|
|
1381
|
+
* repository; `benchmarks/` measures what the writer costs now, which is the useful number.
|
|
600
1382
|
*
|
|
601
1383
|
* @private
|
|
602
1384
|
*/
|
|
603
1385
|
_buildSymbolTable() {
|
|
604
|
-
this.
|
|
1386
|
+
this._symbolCounts = [];
|
|
1387
|
+
this._symbolFacts = [];
|
|
605
1388
|
this._symbolTableMetadata = [];
|
|
606
1389
|
this._symbolIndexByValue = [];
|
|
607
1390
|
if (this._df.columns.length === 0) {
|
|
@@ -614,33 +1397,144 @@ var init_QvdFileWriter = __esm({
|
|
|
614
1397
|
const data = this._df.data;
|
|
615
1398
|
const numColumns = columns.length;
|
|
616
1399
|
const numRows = data.length;
|
|
1400
|
+
validateColumnNames(columns, this._path);
|
|
1401
|
+
const record = normaliseStoredSymbols(this._df.storedSymbols);
|
|
617
1402
|
this._emitProgress("symbol-table", 0, numColumns);
|
|
618
|
-
const
|
|
1403
|
+
const state = columns.map((name) => {
|
|
1404
|
+
const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
|
|
1405
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1406
|
+
let byValue = null;
|
|
1407
|
+
let textByNumber = null;
|
|
1408
|
+
if (entry !== null && valuesAreNumbers(entry)) {
|
|
1409
|
+
textByNumber = firstTextByValue(entry);
|
|
1410
|
+
} else if (entry !== null) {
|
|
1411
|
+
byValue = /* @__PURE__ */ new Map();
|
|
1412
|
+
for (let index = 0; index < entry.values.length; index++) {
|
|
1413
|
+
const found = byValue.get(entry.values[index]);
|
|
1414
|
+
if (found === void 0) {
|
|
1415
|
+
byValue.set(entry.values[index], index);
|
|
1416
|
+
} else if (typeof found === "number") {
|
|
1417
|
+
byValue.set(entry.values[index], [found, index]);
|
|
1418
|
+
} else {
|
|
1419
|
+
found.push(index);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
return {
|
|
1424
|
+
name,
|
|
1425
|
+
keys: [],
|
|
1426
|
+
texts: [],
|
|
1427
|
+
byKey,
|
|
1428
|
+
// A cell is its own key without a record entry, and with one whose values are their numbers, so
|
|
1429
|
+
// one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
|
|
1430
|
+
byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
|
|
1431
|
+
byObject: /* @__PURE__ */ new Map(),
|
|
1432
|
+
entry,
|
|
1433
|
+
textByNumber,
|
|
1434
|
+
byValue,
|
|
1435
|
+
facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
|
|
1436
|
+
};
|
|
1437
|
+
});
|
|
1438
|
+
const byCells = state.map((column) => column.byCell);
|
|
1439
|
+
const byObjects = state.map((column) => column.byObject);
|
|
619
1440
|
const containsNull = columns.map(() => false);
|
|
620
1441
|
for (let row = 0; row < numRows; row++) {
|
|
621
1442
|
const values = data[row];
|
|
1443
|
+
if (values !== null && values !== void 0 && !Array.isArray(values)) {
|
|
1444
|
+
throw new QvdValidationError("Each row must be an array of values", {
|
|
1445
|
+
row,
|
|
1446
|
+
type: typeof values,
|
|
1447
|
+
file: this._path,
|
|
1448
|
+
stage: "buildSymbolTable"
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1451
|
+
if (values !== null && values !== void 0 && values.length > numColumns) {
|
|
1452
|
+
throw new QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
|
|
1453
|
+
row,
|
|
1454
|
+
values: values.length,
|
|
1455
|
+
fields: numColumns,
|
|
1456
|
+
file: this._path,
|
|
1457
|
+
stage: "buildSymbolTable"
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
622
1460
|
for (let column = 0; column < numColumns; column++) {
|
|
623
1461
|
const value = values?.[column];
|
|
624
1462
|
if (value === null || value === void 0) {
|
|
625
1463
|
containsNull[column] = true;
|
|
626
1464
|
continue;
|
|
627
1465
|
}
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
1466
|
+
if (typeof value === "object") {
|
|
1467
|
+
if (!byObjects[column].has(value)) {
|
|
1468
|
+
slotForObject(state[column], value, row, this._path);
|
|
1469
|
+
}
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1472
|
+
if (byCells[column].has(value)) {
|
|
1473
|
+
continue;
|
|
1474
|
+
}
|
|
1475
|
+
if (typeof value === "number") {
|
|
1476
|
+
if (numberProblem(value) !== null) {
|
|
1477
|
+
checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
|
|
1478
|
+
}
|
|
1479
|
+
} else if (typeof value === "string") {
|
|
1480
|
+
if (textProblem(value) !== null) {
|
|
1481
|
+
checkText(value, "A string value", {
|
|
1482
|
+
column: columns[column],
|
|
1483
|
+
row,
|
|
1484
|
+
file: this._path,
|
|
1485
|
+
stage: "buildSymbolTable"
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
} else {
|
|
1489
|
+
refuseCell(value, columns[column], row, this._path);
|
|
631
1490
|
}
|
|
1491
|
+
byCells[column].set(value, slotForCell(state[column], value, row, this._path));
|
|
632
1492
|
}
|
|
633
1493
|
}
|
|
634
1494
|
const columnBuffers = [];
|
|
635
1495
|
let symbolsOffset = 0;
|
|
636
1496
|
for (let column = 0; column < numColumns; column++) {
|
|
637
|
-
const
|
|
638
|
-
const
|
|
1497
|
+
const { keys, texts } = state[column];
|
|
1498
|
+
const kinds = new Uint8Array(keys.length);
|
|
1499
|
+
let byteLength = 0;
|
|
1500
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1501
|
+
const key = keys[slot];
|
|
1502
|
+
const text = texts[slot];
|
|
1503
|
+
if (typeof key === "number") {
|
|
1504
|
+
if (numberProblem(key) !== null) {
|
|
1505
|
+
checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
|
|
1506
|
+
}
|
|
1507
|
+
if (text !== null && textProblem(text) !== null) {
|
|
1508
|
+
checkText(text, "The text of a dual value", {
|
|
1509
|
+
column: columns[column],
|
|
1510
|
+
file: this._path,
|
|
1511
|
+
stage: "buildSymbolTable"
|
|
1512
|
+
});
|
|
1513
|
+
}
|
|
1514
|
+
kinds[slot] = kindOf(key, text);
|
|
1515
|
+
byteLength += symbolByteLength(kinds[slot], key, text);
|
|
1516
|
+
} else {
|
|
1517
|
+
kinds[slot] = kindOf(null, key);
|
|
1518
|
+
byteLength += symbolByteLength(kinds[slot], null, key);
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
const columnBuffer = Buffer.allocUnsafe(byteLength);
|
|
1522
|
+
let offset = 0;
|
|
1523
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1524
|
+
const key = keys[slot];
|
|
1525
|
+
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1526
|
+
}
|
|
1527
|
+
assert2(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
639
1528
|
columnBuffers.push(columnBuffer);
|
|
640
|
-
this._symbolTableMetadata?.push([symbolsOffset,
|
|
641
|
-
this.
|
|
642
|
-
this.
|
|
643
|
-
|
|
1529
|
+
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1530
|
+
this._symbolCounts?.push(keys.length);
|
|
1531
|
+
this._symbolFacts?.push(state[column].facts);
|
|
1532
|
+
this._symbolIndexByValue?.push({
|
|
1533
|
+
byCell: state[column].byCell,
|
|
1534
|
+
byKey: state[column].byKey,
|
|
1535
|
+
byObject: state[column].byObject
|
|
1536
|
+
});
|
|
1537
|
+
symbolsOffset += byteLength;
|
|
644
1538
|
this._emitProgress("symbol-table", column + 1, numColumns);
|
|
645
1539
|
}
|
|
646
1540
|
this._symbolBuffer = Buffer.concat(columnBuffers);
|
|
@@ -669,7 +1563,7 @@ var init_QvdFileWriter = __esm({
|
|
|
669
1563
|
* @private
|
|
670
1564
|
*/
|
|
671
1565
|
_buildIndexTable() {
|
|
672
|
-
assert2(this.
|
|
1566
|
+
assert2(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
673
1567
|
assert2(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
674
1568
|
assert2(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
675
1569
|
this._indexTableMetadata = [];
|
|
@@ -683,20 +1577,24 @@ var init_QvdFileWriter = __esm({
|
|
|
683
1577
|
let totalBits = 0;
|
|
684
1578
|
for (let column = 0; column < numColumns; column++) {
|
|
685
1579
|
const fieldContainsNull = this._symbolTableMetadata[column][2];
|
|
686
|
-
const symbolCount = this.
|
|
1580
|
+
const symbolCount = this._symbolCounts[column];
|
|
687
1581
|
const nullShift = fieldContainsNull ? 2 : 0;
|
|
688
1582
|
const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
|
|
689
1583
|
const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
1584
|
+
const lookup = this._symbolIndexByValue[column];
|
|
1585
|
+
const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
|
|
1586
|
+
for (const map of maps) {
|
|
1587
|
+
for (const index of map.values()) {
|
|
1588
|
+
if (index + nullShift > maxStoredIndex) {
|
|
1589
|
+
throw new QvdValidationError("The symbol table and the index table are out of sync", {
|
|
1590
|
+
field: columns[column],
|
|
1591
|
+
storedIndex: index + nullShift,
|
|
1592
|
+
maxStoredIndex,
|
|
1593
|
+
symbolCount,
|
|
1594
|
+
file: this._path,
|
|
1595
|
+
stage: "buildIndexTable"
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
700
1598
|
}
|
|
701
1599
|
}
|
|
702
1600
|
layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
|
|
@@ -707,6 +1605,9 @@ var init_QvdFileWriter = __esm({
|
|
|
707
1605
|
this._recordByteSize = recordByteSize;
|
|
708
1606
|
this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
|
|
709
1607
|
const progressInterval = Math.max(1, Math.floor(numRows / 100));
|
|
1608
|
+
const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
|
|
1609
|
+
const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
|
|
1610
|
+
const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
|
|
710
1611
|
for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
|
|
711
1612
|
const values = data[row];
|
|
712
1613
|
for (let column = 0; column < numColumns; column++) {
|
|
@@ -714,7 +1615,7 @@ var init_QvdFileWriter = __esm({
|
|
|
714
1615
|
if (value === null || value === void 0) {
|
|
715
1616
|
continue;
|
|
716
1617
|
}
|
|
717
|
-
const index =
|
|
1618
|
+
const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
|
|
718
1619
|
if (index === void 0) {
|
|
719
1620
|
throw new QvdValidationError("A value is missing from the symbol table", {
|
|
720
1621
|
field: columns[column],
|
|
@@ -731,29 +1632,6 @@ var init_QvdFileWriter = __esm({
|
|
|
731
1632
|
}
|
|
732
1633
|
this._symbolIndexByValue = null;
|
|
733
1634
|
}
|
|
734
|
-
/**
|
|
735
|
-
* Converts a raw value/literal to a QVD symbol.
|
|
736
|
-
*
|
|
737
|
-
* @param {any} raw The raw value/literal to convert.
|
|
738
|
-
* @return {QvdSymbol|null} The converted QVD symbol.
|
|
739
|
-
*/
|
|
740
|
-
static _convertRawToSymbol(raw) {
|
|
741
|
-
if (raw === null || raw === void 0) {
|
|
742
|
-
return null;
|
|
743
|
-
}
|
|
744
|
-
const INT32_MIN = -2147483648;
|
|
745
|
-
const INT32_MAX = 2147483647;
|
|
746
|
-
const isInteger = typeof raw === "number" && Number.isInteger(raw);
|
|
747
|
-
const isFloat = typeof raw === "number" && !Number.isInteger(raw);
|
|
748
|
-
const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
|
|
749
|
-
if (isInteger && isWithinInt32Range) {
|
|
750
|
-
return QvdSymbol.fromDualIntValue(raw, raw.toString());
|
|
751
|
-
} else if (isFloat || isInteger && !isWithinInt32Range) {
|
|
752
|
-
return QvdSymbol.fromDualDoubleValue(raw, raw.toString());
|
|
753
|
-
} else {
|
|
754
|
-
return QvdSymbol.fromStringValue(raw);
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
1635
|
/**
|
|
758
1636
|
* Persists the data frame to a QVD file.
|
|
759
1637
|
*/
|
|
@@ -807,11 +1685,12 @@ function estimateRowMemory(rows, columnCount) {
|
|
|
807
1685
|
}
|
|
808
1686
|
return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
|
|
809
1687
|
}
|
|
810
|
-
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
1688
|
+
function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true, rowsLive = null) {
|
|
811
1689
|
const FULL_PARSE_OVERHEAD = 6;
|
|
812
1690
|
const MINIMAL_OVERHEAD = 0.01;
|
|
813
1691
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
814
|
-
const
|
|
1692
|
+
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
1693
|
+
const rowMemory = materialisesRows ? estimateRowMemory(liveRows, columnCount) : BASE_BYTES;
|
|
815
1694
|
if (maxRows === null || maxRows >= totalRows) {
|
|
816
1695
|
return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
|
|
817
1696
|
}
|
|
@@ -821,18 +1700,44 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
821
1700
|
const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
|
|
822
1701
|
return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
|
|
823
1702
|
}
|
|
824
|
-
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true) {
|
|
825
|
-
|
|
1703
|
+
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
|
|
1704
|
+
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
1705
|
+
if (costOf(totalRows) <= budget) {
|
|
826
1706
|
return totalRows;
|
|
827
1707
|
}
|
|
828
|
-
if (
|
|
1708
|
+
if (costOf(0) > budget) {
|
|
829
1709
|
return 0;
|
|
830
1710
|
}
|
|
831
1711
|
let low = 0;
|
|
832
1712
|
let high = totalRows;
|
|
833
1713
|
while (high - low > 1) {
|
|
834
1714
|
const mid = Math.floor((low + high) / 2);
|
|
835
|
-
if (
|
|
1715
|
+
if (costOf(mid) <= budget) {
|
|
1716
|
+
low = mid;
|
|
1717
|
+
} else {
|
|
1718
|
+
high = mid;
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
return low;
|
|
1722
|
+
}
|
|
1723
|
+
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
|
|
1724
|
+
const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
|
|
1725
|
+
const fits = /* @__PURE__ */ __name((chunk) => {
|
|
1726
|
+
const live = Math.min(chunk * liveRowsPerChunk, covered);
|
|
1727
|
+
const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
|
|
1728
|
+
return cost <= budget;
|
|
1729
|
+
}, "fits");
|
|
1730
|
+
if (fits(covered)) {
|
|
1731
|
+
return covered;
|
|
1732
|
+
}
|
|
1733
|
+
if (!fits(1)) {
|
|
1734
|
+
return 0;
|
|
1735
|
+
}
|
|
1736
|
+
let low = 1;
|
|
1737
|
+
let high = covered;
|
|
1738
|
+
while (high - low > 1) {
|
|
1739
|
+
const mid = Math.floor((low + high) / 2);
|
|
1740
|
+
if (fits(mid)) {
|
|
836
1741
|
low = mid;
|
|
837
1742
|
} else {
|
|
838
1743
|
high = mid;
|
|
@@ -840,7 +1745,7 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
|
|
|
840
1745
|
}
|
|
841
1746
|
return low;
|
|
842
1747
|
}
|
|
843
|
-
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true) {
|
|
1748
|
+
function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true, live = null) {
|
|
844
1749
|
if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
|
|
845
1750
|
throw new QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
|
|
846
1751
|
}
|
|
@@ -849,12 +1754,16 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
849
1754
|
}
|
|
850
1755
|
const budget = getMemoryBudget();
|
|
851
1756
|
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
852
|
-
const
|
|
853
|
-
const
|
|
1757
|
+
const rowsLive = live === null ? null : live.rows;
|
|
1758
|
+
const liveRowsPerChunk = live === null ? 1 : live.perChunk;
|
|
1759
|
+
const liveRows = rowsLive === null ? rowsToLoad : Math.min(rowsLive, rowsToLoad);
|
|
1760
|
+
const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows, rowsLive);
|
|
1761
|
+
const externalMemory = estimateExternalMemory(liveRows, columnCount);
|
|
854
1762
|
const bounded = budget.candidates.map((candidate) => {
|
|
855
1763
|
const heapOnly = candidate.source === "V8 heap limit";
|
|
856
1764
|
return {
|
|
857
1765
|
...candidate,
|
|
1766
|
+
heapOnly,
|
|
858
1767
|
needs: heapOnly ? heapMemory : heapMemory + externalMemory,
|
|
859
1768
|
allowed: candidate.bytes * safetyFactor,
|
|
860
1769
|
bounds: heapOnly ? "the V8 heap" : "the whole process"
|
|
@@ -870,12 +1779,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
870
1779
|
const estimatedMemory = binding ? binding.needs : heapMemory;
|
|
871
1780
|
const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
|
|
872
1781
|
if (binding) {
|
|
1782
|
+
const includeExternal = !binding.heapOnly;
|
|
873
1783
|
const recommendedMaxRows = recommendedRowsFor(
|
|
874
1784
|
maxAllowedMemory,
|
|
875
1785
|
symbolTableSize,
|
|
876
1786
|
totalRows,
|
|
877
1787
|
columnCount,
|
|
878
|
-
materialisesRows
|
|
1788
|
+
materialisesRows,
|
|
1789
|
+
includeExternal
|
|
879
1790
|
);
|
|
880
1791
|
const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
|
|
881
1792
|
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
@@ -887,15 +1798,27 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
887
1798
|
const limitingScope = binding.bounds;
|
|
888
1799
|
const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
|
|
889
1800
|
const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
|
|
890
|
-
const nothingFits = recommendedMaxRows === 0;
|
|
891
1801
|
const containerBound = binding.source === "container memory limit";
|
|
1802
|
+
const chunked = rowsLive !== null;
|
|
1803
|
+
const recommendedChunk = chunked ? recommendedChunkFor(
|
|
1804
|
+
maxAllowedMemory,
|
|
1805
|
+
symbolTableSize,
|
|
1806
|
+
maxRows,
|
|
1807
|
+
totalRows,
|
|
1808
|
+
columnCount,
|
|
1809
|
+
liveRowsPerChunk,
|
|
1810
|
+
includeExternal
|
|
1811
|
+
) : 0;
|
|
1812
|
+
const knob = chunked ? "chunkSize" : "limit";
|
|
1813
|
+
const recommendedValue = chunked ? recommendedChunk : recommendedMaxRows;
|
|
1814
|
+
const nothingFits = recommendedValue === 0;
|
|
892
1815
|
let advice;
|
|
893
1816
|
if (nothingFits) {
|
|
894
|
-
advice = `No row count fits this budget - the symbol table alone exceeds it, so
|
|
1817
|
+
advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
|
|
895
1818
|
} else if (containerBound) {
|
|
896
|
-
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or
|
|
1819
|
+
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
|
|
897
1820
|
} else {
|
|
898
|
-
advice = `Try
|
|
1821
|
+
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
899
1822
|
}
|
|
900
1823
|
throw new QvdValidationError(
|
|
901
1824
|
`Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
@@ -915,32 +1838,55 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
915
1838
|
columnCount,
|
|
916
1839
|
totalRows,
|
|
917
1840
|
maxRows,
|
|
918
|
-
recommendedMaxRows
|
|
1841
|
+
recommendedMaxRows,
|
|
1842
|
+
// Only present when a chunk size is what overflowed, so a caller cannot mistake one
|
|
1843
|
+
// recommendation for the other.
|
|
1844
|
+
...chunked ? { rowsLive, recommendedChunkSize: recommendedChunk } : {}
|
|
919
1845
|
}
|
|
920
1846
|
);
|
|
921
1847
|
}
|
|
922
1848
|
}
|
|
1849
|
+
function formatCount(value) {
|
|
1850
|
+
return value.toLocaleString("en-US");
|
|
1851
|
+
}
|
|
923
1852
|
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
924
1853
|
const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
|
|
925
|
-
if (symbolTableSize
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
);
|
|
1854
|
+
if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
|
|
1858
|
+
const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
|
|
1859
|
+
if (estimatedMemory <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
1860
|
+
return;
|
|
933
1861
|
}
|
|
1862
|
+
const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
|
|
1863
|
+
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
1864
|
+
const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
|
|
1865
|
+
console.warn(
|
|
1866
|
+
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
1867
|
+
);
|
|
934
1868
|
}
|
|
935
1869
|
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
936
1870
|
var init_memoryUtils = __esm({
|
|
937
1871
|
"src/util/memoryUtils.js"() {
|
|
938
1872
|
init_QvdErrors();
|
|
1873
|
+
__name(getHeapLimit, "getHeapLimit");
|
|
1874
|
+
__name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
|
|
1875
|
+
__name(getMemoryBudget, "getMemoryBudget");
|
|
939
1876
|
HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
|
|
940
1877
|
MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
1878
|
+
__name(usableOldSpaceLimit, "usableOldSpaceLimit");
|
|
941
1879
|
BASE_BYTES = 16 * 1024 * 1024;
|
|
942
1880
|
ROW_BASE_BYTES = 72;
|
|
943
1881
|
PER_CELL_BYTES = 8;
|
|
1882
|
+
__name(estimateExternalMemory, "estimateExternalMemory");
|
|
1883
|
+
__name(estimateRowMemory, "estimateRowMemory");
|
|
1884
|
+
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
1885
|
+
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
1886
|
+
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
1887
|
+
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
1888
|
+
__name(formatCount, "formatCount");
|
|
1889
|
+
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
944
1890
|
}
|
|
945
1891
|
});
|
|
946
1892
|
|
|
@@ -971,7 +1917,7 @@ function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
|
|
|
971
1917
|
const maxMB = Math.round(MAX_SYMBOL_TABLE_SIZE / 1024 / 1024);
|
|
972
1918
|
const heapMB = Math.round(heapLimit / 1024 / 1024);
|
|
973
1919
|
throw new QvdValidationError(
|
|
974
|
-
`Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without maxRows, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
|
|
1920
|
+
`Symbol table too large (${sizeMB}MB exceeds ${maxMB}MB limit for lazy loading). This QVD file contains extremely high-cardinality fields. Limit scales with heap size (current: ${heapMB}MB, limit: 12.5% = ${maxMB}MB). Consider: (1) loading the full file without a row window - maxRows, limit or offset - since the symbol table is read in full either way, (2) increasing heap size with --max-old-space-size, or (3) aggregating high-cardinality fields.`,
|
|
975
1921
|
{
|
|
976
1922
|
file: filePath,
|
|
977
1923
|
symbolTableSize: symbolTableLength,
|
|
@@ -1045,7 +1991,7 @@ function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
|
|
|
1045
1991
|
});
|
|
1046
1992
|
}
|
|
1047
1993
|
}
|
|
1048
|
-
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
|
|
1994
|
+
function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null, windowFirstRow = 0, bufferFirstRow = 0) {
|
|
1049
1995
|
if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
|
|
1050
1996
|
throw new QvdCorruptedError("Invalid record byte size", {
|
|
1051
1997
|
recordSize,
|
|
@@ -1107,23 +2053,28 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
|
|
|
1107
2053
|
}
|
|
1108
2054
|
}
|
|
1109
2055
|
const requiredIndexBytes = rowsToLoad * recordSize;
|
|
1110
|
-
|
|
2056
|
+
const bufferRecordStart = (windowFirstRow - bufferFirstRow) * recordSize;
|
|
2057
|
+
if (indexTableOffset + bufferRecordStart + requiredIndexBytes > bufferLength) {
|
|
1111
2058
|
throw new QvdCorruptedError("Index table truncated", {
|
|
1112
2059
|
indexTableOffset,
|
|
1113
2060
|
requiredBytes: requiredIndexBytes,
|
|
1114
|
-
availableBytes: Math.max(0, bufferLength - indexTableOffset),
|
|
2061
|
+
availableBytes: Math.max(0, bufferLength - indexTableOffset - bufferRecordStart),
|
|
1115
2062
|
rowsToLoad,
|
|
2063
|
+
windowFirstRow,
|
|
2064
|
+
bufferFirstRow,
|
|
1116
2065
|
recordSize,
|
|
1117
2066
|
bufferSize: bufferLength,
|
|
1118
2067
|
file: filePath,
|
|
1119
2068
|
stage: "parseIndexTable"
|
|
1120
2069
|
});
|
|
1121
2070
|
}
|
|
1122
|
-
|
|
2071
|
+
const requiredTableBytes = (windowFirstRow + rowsToLoad) * recordSize;
|
|
2072
|
+
if (indexTableLength < requiredTableBytes) {
|
|
1123
2073
|
throw new QvdCorruptedError("Index table length smaller than required", {
|
|
1124
2074
|
indexTableLength,
|
|
1125
|
-
requiredBytes:
|
|
2075
|
+
requiredBytes: requiredTableBytes,
|
|
1126
2076
|
rowsToLoad,
|
|
2077
|
+
windowFirstRow,
|
|
1127
2078
|
recordSize,
|
|
1128
2079
|
file: filePath,
|
|
1129
2080
|
stage: "parseIndexTable"
|
|
@@ -1184,262 +2135,235 @@ var init_validationUtils = __esm({
|
|
|
1184
2135
|
init_QvdErrors();
|
|
1185
2136
|
init_memoryUtils();
|
|
1186
2137
|
init_bitUtils();
|
|
2138
|
+
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2139
|
+
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2140
|
+
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2141
|
+
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2142
|
+
__name(validateRecordCount, "validateRecordCount");
|
|
2143
|
+
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2144
|
+
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
1187
2145
|
}
|
|
1188
2146
|
});
|
|
1189
2147
|
|
|
1190
2148
|
// src/util/symbolParser.js
|
|
1191
|
-
function
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
bufferSize: bufferLength,
|
|
1197
|
-
file: filePath,
|
|
1198
|
-
stage: "parseSymbolTable"
|
|
1199
|
-
});
|
|
1200
|
-
}
|
|
1201
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1202
|
-
const value = Buffer.from(byteData).readIntLE(0, byteData.length);
|
|
1203
|
-
return { symbol: QvdSymbol.fromIntValue(value), bytesRead: 4 };
|
|
1204
|
-
}
|
|
1205
|
-
function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1206
|
-
if (pointer + 8 > bufferLength) {
|
|
1207
|
-
throw new QvdCorruptedError("Buffer overflow reading double symbol", {
|
|
1208
|
-
field: fieldName,
|
|
1209
|
-
pointer,
|
|
1210
|
-
bufferSize: bufferLength,
|
|
1211
|
-
file: filePath,
|
|
1212
|
-
stage: "parseSymbolTable"
|
|
1213
|
-
});
|
|
1214
|
-
}
|
|
1215
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1216
|
-
const value = Buffer.from(byteData).readDoubleLE(0);
|
|
1217
|
-
return { symbol: QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
|
|
1218
|
-
}
|
|
1219
|
-
function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1220
|
-
const startPointer = pointer;
|
|
1221
|
-
const maxStringLength = 1048576;
|
|
1222
|
-
let stringLength = 0;
|
|
1223
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1224
|
-
if (stringLength >= maxStringLength) {
|
|
1225
|
-
throw new QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1226
|
-
field: fieldName,
|
|
1227
|
-
maxLength: maxStringLength,
|
|
1228
|
-
file: filePath,
|
|
1229
|
-
stage: "parseSymbolTable"
|
|
1230
|
-
});
|
|
1231
|
-
}
|
|
1232
|
-
pointer++;
|
|
1233
|
-
stringLength++;
|
|
1234
|
-
}
|
|
1235
|
-
if (pointer >= bufferLength) {
|
|
1236
|
-
throw new QvdCorruptedError("String symbol not null-terminated", {
|
|
2149
|
+
function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
|
|
2150
|
+
const bufferLength = symbolBuffer.length;
|
|
2151
|
+
const found = symbolBuffer.indexOf(0, from);
|
|
2152
|
+
if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
|
|
2153
|
+
throw new QvdCorruptedError(`${kind} exceeds maximum length`, {
|
|
1237
2154
|
field: fieldName,
|
|
1238
|
-
|
|
1239
|
-
bufferSize: bufferLength,
|
|
2155
|
+
maxLength: MAX_TEXT_BYTES,
|
|
1240
2156
|
file: filePath,
|
|
1241
2157
|
stage: "parseSymbolTable"
|
|
1242
2158
|
});
|
|
1243
2159
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
}
|
|
1247
|
-
function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1248
|
-
const startPointer = pointer;
|
|
1249
|
-
const maxStringLength = 1048576;
|
|
1250
|
-
let stringLength = 0;
|
|
1251
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1252
|
-
if (stringLength >= maxStringLength) {
|
|
1253
|
-
throw new QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1254
|
-
field: fieldName,
|
|
1255
|
-
maxLength: maxStringLength,
|
|
1256
|
-
file: filePath,
|
|
1257
|
-
stage: "parseSymbolTable"
|
|
1258
|
-
});
|
|
1259
|
-
}
|
|
1260
|
-
pointer++;
|
|
1261
|
-
stringLength++;
|
|
1262
|
-
}
|
|
1263
|
-
if (pointer >= bufferLength) {
|
|
1264
|
-
throw new QvdCorruptedError("String symbol not null-terminated", {
|
|
2160
|
+
if (found === -1) {
|
|
2161
|
+
throw new QvdCorruptedError(`${kind} not null-terminated`, {
|
|
1265
2162
|
field: fieldName,
|
|
1266
|
-
pointer,
|
|
2163
|
+
pointer: bufferLength,
|
|
1267
2164
|
bufferSize: bufferLength,
|
|
1268
2165
|
file: filePath,
|
|
1269
2166
|
stage: "parseSymbolTable"
|
|
1270
2167
|
});
|
|
1271
2168
|
}
|
|
1272
|
-
return
|
|
2169
|
+
return found;
|
|
1273
2170
|
}
|
|
1274
|
-
function
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
});
|
|
1283
|
-
}
|
|
1284
|
-
const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1285
|
-
const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
|
|
1286
|
-
pointer += 4;
|
|
1287
|
-
const stringStart = pointer;
|
|
1288
|
-
const maxStringLength = 1048576;
|
|
1289
|
-
let stringLength = 0;
|
|
1290
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1291
|
-
if (stringLength >= maxStringLength) {
|
|
1292
|
-
throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1293
|
-
field: fieldName,
|
|
1294
|
-
maxLength: maxStringLength,
|
|
1295
|
-
file: filePath,
|
|
1296
|
-
stage: "parseSymbolTable"
|
|
1297
|
-
});
|
|
1298
|
-
}
|
|
1299
|
-
pointer++;
|
|
1300
|
-
stringLength++;
|
|
1301
|
-
}
|
|
1302
|
-
if (pointer >= bufferLength) {
|
|
1303
|
-
throw new QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1304
|
-
field: fieldName,
|
|
1305
|
-
pointer,
|
|
1306
|
-
bufferSize: bufferLength,
|
|
1307
|
-
file: filePath,
|
|
1308
|
-
stage: "parseSymbolTable"
|
|
1309
|
-
});
|
|
1310
|
-
}
|
|
1311
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1312
|
-
return { symbol: QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
|
|
2171
|
+
function overflow(message, pointer, bufferLength, fieldName, filePath) {
|
|
2172
|
+
throw new QvdCorruptedError(message, {
|
|
2173
|
+
field: fieldName,
|
|
2174
|
+
pointer,
|
|
2175
|
+
bufferSize: bufferLength,
|
|
2176
|
+
file: filePath,
|
|
2177
|
+
stage: "parseSymbolTable"
|
|
2178
|
+
});
|
|
1313
2179
|
}
|
|
1314
|
-
function
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
2180
|
+
function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
|
|
2181
|
+
const bufferLength = symbolBuffer.length;
|
|
2182
|
+
const numbers = [];
|
|
2183
|
+
const texts = [];
|
|
2184
|
+
let pointer = start;
|
|
2185
|
+
while (pointer < end) {
|
|
2186
|
+
const typeByte = symbolBuffer[pointer++];
|
|
2187
|
+
const decode = keep === null || keep.has(numbers.length);
|
|
2188
|
+
let number = null;
|
|
2189
|
+
let text = null;
|
|
2190
|
+
switch (typeByte) {
|
|
2191
|
+
case 1: {
|
|
2192
|
+
if (decode) {
|
|
2193
|
+
if (pointer + 4 > bufferLength) {
|
|
2194
|
+
overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
|
|
2195
|
+
}
|
|
2196
|
+
number = symbolBuffer.readInt32LE(pointer);
|
|
2197
|
+
}
|
|
2198
|
+
pointer += 4;
|
|
2199
|
+
break;
|
|
2200
|
+
}
|
|
2201
|
+
case 2: {
|
|
2202
|
+
if (decode) {
|
|
2203
|
+
if (pointer + 8 > bufferLength) {
|
|
2204
|
+
overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
|
|
2205
|
+
}
|
|
2206
|
+
number = symbolBuffer.readDoubleLE(pointer);
|
|
2207
|
+
}
|
|
2208
|
+
pointer += 8;
|
|
2209
|
+
break;
|
|
2210
|
+
}
|
|
2211
|
+
case 4: {
|
|
2212
|
+
const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
|
|
2213
|
+
if (decode) {
|
|
2214
|
+
text = symbolBuffer.toString("utf8", pointer, terminator);
|
|
2215
|
+
}
|
|
2216
|
+
pointer = terminator + 1;
|
|
2217
|
+
break;
|
|
2218
|
+
}
|
|
2219
|
+
case 5:
|
|
2220
|
+
case 6: {
|
|
2221
|
+
const numberBytes = typeByte === 5 ? 4 : 8;
|
|
2222
|
+
if (pointer + numberBytes > bufferLength) {
|
|
2223
|
+
const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
|
|
2224
|
+
overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
|
|
2225
|
+
}
|
|
2226
|
+
const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
|
|
2227
|
+
if (decode) {
|
|
2228
|
+
number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
|
|
2229
|
+
text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
|
|
2230
|
+
}
|
|
2231
|
+
pointer = terminator + 1;
|
|
2232
|
+
break;
|
|
2233
|
+
}
|
|
2234
|
+
default: {
|
|
2235
|
+
throw new QvdParseError("Unknown symbol type byte", {
|
|
2236
|
+
typeByte: typeByte.toString(16),
|
|
2237
|
+
offset: pointer - 1,
|
|
2238
|
+
file: filePath,
|
|
2239
|
+
stage: "parseSymbolTable"
|
|
2240
|
+
});
|
|
2241
|
+
}
|
|
1338
2242
|
}
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
}
|
|
1342
|
-
if (pointer >= bufferLength) {
|
|
1343
|
-
throw new QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1344
|
-
field: fieldName,
|
|
1345
|
-
pointer,
|
|
1346
|
-
bufferSize: bufferLength,
|
|
1347
|
-
file: filePath,
|
|
1348
|
-
stage: "parseSymbolTable"
|
|
1349
|
-
});
|
|
2243
|
+
numbers.push(number);
|
|
2244
|
+
texts.push(text);
|
|
1350
2245
|
}
|
|
1351
|
-
|
|
1352
|
-
return {
|
|
1353
|
-
symbol: QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
|
|
1354
|
-
bytesRead: pointer - (stringStart - 8) + 1
|
|
1355
|
-
};
|
|
2246
|
+
return { numbers, texts };
|
|
1356
2247
|
}
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
});
|
|
2248
|
+
var MAX_TEXT_BYTES;
|
|
2249
|
+
var init_symbolParser = __esm({
|
|
2250
|
+
"src/util/symbolParser.js"() {
|
|
2251
|
+
init_QvdErrors();
|
|
2252
|
+
MAX_TEXT_BYTES = 1048576;
|
|
2253
|
+
__name(textEnd, "textEnd");
|
|
2254
|
+
__name(overflow, "overflow");
|
|
2255
|
+
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
1366
2256
|
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
2257
|
+
});
|
|
2258
|
+
|
|
2259
|
+
// src/util/resolveSymbols.js
|
|
2260
|
+
function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
|
|
2261
|
+
const length = symbols.numbers.length;
|
|
2262
|
+
const values = new Array(length);
|
|
2263
|
+
const entryValues = [];
|
|
2264
|
+
const numbers = [];
|
|
2265
|
+
const texts = [];
|
|
2266
|
+
let pure = 0;
|
|
2267
|
+
let partial = false;
|
|
2268
|
+
for (let index = 0; index < length; index++) {
|
|
2269
|
+
const text = symbols.texts[index];
|
|
2270
|
+
const number = symbols.numbers[index];
|
|
2271
|
+
if (text === null) {
|
|
2272
|
+
values[index] = number === null ? void 0 : number;
|
|
2273
|
+
if (number !== null) pure++;
|
|
2274
|
+
continue;
|
|
1379
2275
|
}
|
|
1380
|
-
|
|
1381
|
-
|
|
2276
|
+
if (number === null) {
|
|
2277
|
+
if (coerce && isNumericText(text)) {
|
|
2278
|
+
values[index] = Number(text);
|
|
2279
|
+
entryValues.push(values[index]);
|
|
2280
|
+
numbers.push(null);
|
|
2281
|
+
texts.push(text);
|
|
2282
|
+
partial = true;
|
|
2283
|
+
} else {
|
|
2284
|
+
values[index] = text;
|
|
2285
|
+
pure++;
|
|
2286
|
+
}
|
|
2287
|
+
continue;
|
|
2288
|
+
}
|
|
2289
|
+
partial = true;
|
|
2290
|
+
if (mode === "both") {
|
|
2291
|
+
values[index] = dualFromSymbol(number, text);
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
|
|
2295
|
+
entryValues.push(values[index]);
|
|
2296
|
+
numbers.push(number);
|
|
2297
|
+
texts.push(text);
|
|
1382
2298
|
}
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2299
|
+
let entry = null;
|
|
2300
|
+
if (entryValues.length > 0) {
|
|
2301
|
+
entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
|
|
2302
|
+
field,
|
|
2303
|
+
values: Object.freeze(entryValues),
|
|
2304
|
+
numbers: Object.freeze(numbers),
|
|
2305
|
+
texts: Object.freeze(texts)
|
|
1390
2306
|
});
|
|
1391
2307
|
}
|
|
1392
|
-
return
|
|
2308
|
+
return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
|
|
1393
2309
|
}
|
|
1394
|
-
function
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
return parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1399
|
-
}
|
|
1400
|
-
return { symbol: null, bytesRead: 4 };
|
|
2310
|
+
function collides(symbols, values, recorded) {
|
|
2311
|
+
for (let index = 0; index < values.length; index++) {
|
|
2312
|
+
if (!isPure(symbols, index, values[index])) {
|
|
2313
|
+
continue;
|
|
1401
2314
|
}
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
return parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1405
|
-
}
|
|
1406
|
-
return { symbol: null, bytesRead: 8 };
|
|
1407
|
-
}
|
|
1408
|
-
case 4: {
|
|
1409
|
-
if (shouldParse) {
|
|
1410
|
-
return parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1411
|
-
}
|
|
1412
|
-
const bytesRead = skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath);
|
|
1413
|
-
return { symbol: null, bytesRead };
|
|
2315
|
+
if (recorded.has(values[index])) {
|
|
2316
|
+
return true;
|
|
1414
2317
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
2318
|
+
}
|
|
2319
|
+
return false;
|
|
2320
|
+
}
|
|
2321
|
+
function isPure(symbols, index, value) {
|
|
2322
|
+
const text = symbols.texts[index];
|
|
2323
|
+
const number = symbols.numbers[index];
|
|
2324
|
+
return text === null ? number !== null : number === null && value === text;
|
|
2325
|
+
}
|
|
2326
|
+
function collisionEntry(symbols, field, values, mode) {
|
|
2327
|
+
const recorded = /* @__PURE__ */ new Set();
|
|
2328
|
+
const inPassA = values.map((value, index) => {
|
|
2329
|
+
if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
|
|
2330
|
+
return false;
|
|
1421
2331
|
}
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
}
|
|
1426
|
-
const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
|
|
1427
|
-
return { symbol: null, bytesRead };
|
|
2332
|
+
const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
|
|
2333
|
+
if (recordedHere) {
|
|
2334
|
+
recorded.add(values[index]);
|
|
1428
2335
|
}
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
2336
|
+
return recordedHere;
|
|
2337
|
+
});
|
|
2338
|
+
const entryValues = [];
|
|
2339
|
+
const numbers = [];
|
|
2340
|
+
const texts = [];
|
|
2341
|
+
for (let index = 0; index < values.length; index++) {
|
|
2342
|
+
if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
|
|
2343
|
+
entryValues.push(values[index]);
|
|
2344
|
+
numbers.push(symbols.numbers[index]);
|
|
2345
|
+
texts.push(symbols.texts[index]);
|
|
1436
2346
|
}
|
|
1437
2347
|
}
|
|
2348
|
+
return Object.freeze({
|
|
2349
|
+
field,
|
|
2350
|
+
values: Object.freeze(entryValues),
|
|
2351
|
+
numbers: Object.freeze(numbers),
|
|
2352
|
+
texts: Object.freeze(texts)
|
|
2353
|
+
});
|
|
1438
2354
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
2355
|
+
function symbolHalves(symbols) {
|
|
2356
|
+
return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
|
|
2357
|
+
}
|
|
2358
|
+
var init_resolveSymbols = __esm({
|
|
2359
|
+
"src/util/resolveSymbols.js"() {
|
|
2360
|
+
init_QvdDual();
|
|
2361
|
+
init_cellRules();
|
|
2362
|
+
__name(resolveFieldSymbols, "resolveFieldSymbols");
|
|
2363
|
+
__name(collides, "collides");
|
|
2364
|
+
__name(isPure, "isPure");
|
|
2365
|
+
__name(collisionEntry, "collisionEntry");
|
|
2366
|
+
__name(symbolHalves, "symbolHalves");
|
|
1443
2367
|
}
|
|
1444
2368
|
});
|
|
1445
2369
|
|
|
@@ -1449,20 +2373,49 @@ __export(QvdColumnTable_exports, {
|
|
|
1449
2373
|
QvdColumn: () => QvdColumn,
|
|
1450
2374
|
QvdColumnTable: () => QvdColumnTable
|
|
1451
2375
|
});
|
|
2376
|
+
function numberOf(value, number) {
|
|
2377
|
+
if (typeof value === "number") {
|
|
2378
|
+
return value;
|
|
2379
|
+
}
|
|
2380
|
+
if (typeof value === "string") {
|
|
2381
|
+
return number ?? NaN;
|
|
2382
|
+
}
|
|
2383
|
+
const dual = asDual(value);
|
|
2384
|
+
return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
|
|
2385
|
+
}
|
|
2386
|
+
function textOf(value) {
|
|
2387
|
+
if (typeof value === "string") {
|
|
2388
|
+
return value;
|
|
2389
|
+
}
|
|
2390
|
+
const dual = asDual(value);
|
|
2391
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
2392
|
+
}
|
|
1452
2393
|
var QvdColumn, QvdColumnTable;
|
|
1453
2394
|
var init_QvdColumnTable = __esm({
|
|
1454
2395
|
"src/QvdColumnTable.js"() {
|
|
1455
2396
|
init_QvdErrors();
|
|
2397
|
+
init_cellRules();
|
|
2398
|
+
init_readOptions();
|
|
2399
|
+
__name(numberOf, "numberOf");
|
|
2400
|
+
__name(textOf, "textOf");
|
|
1456
2401
|
QvdColumn = class {
|
|
2402
|
+
static {
|
|
2403
|
+
__name(this, "QvdColumn");
|
|
2404
|
+
}
|
|
1457
2405
|
/**
|
|
1458
2406
|
* @param {string} name The field name.
|
|
1459
2407
|
* @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
|
|
1460
2408
|
* @param {Array<any>} symbols The field's distinct values, indexed by code.
|
|
2409
|
+
* @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
|
|
2410
|
+
* field whose values do not show them all - a dual read as one half, a string read as a number.
|
|
2411
|
+
* Without them, the halves are derived from the values: a string is its own text, a dual has its
|
|
2412
|
+
* own, and a number has none.
|
|
1461
2413
|
*/
|
|
1462
|
-
constructor(name, codes, symbols) {
|
|
2414
|
+
constructor(name, codes, symbols, halves = null) {
|
|
1463
2415
|
this._name = name;
|
|
1464
2416
|
this._codes = codes;
|
|
1465
2417
|
this._symbols = symbols;
|
|
2418
|
+
this._halves = halves;
|
|
1466
2419
|
Object.freeze(this);
|
|
1467
2420
|
}
|
|
1468
2421
|
/** @return {string} The field name. */
|
|
@@ -1489,6 +2442,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1489
2442
|
*
|
|
1490
2443
|
* One entry per distinct value, not per row: a few thousand entries for a column of millions.
|
|
1491
2444
|
*
|
|
2445
|
+
* A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
|
|
2446
|
+
* entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
|
|
2447
|
+
*
|
|
1492
2448
|
* @return {ReadonlyArray<any>} The dictionary.
|
|
1493
2449
|
*/
|
|
1494
2450
|
get symbols() {
|
|
@@ -1512,6 +2468,42 @@ var init_QvdColumnTable = __esm({
|
|
|
1512
2468
|
const code = this._codes[row];
|
|
1513
2469
|
return code < 0 ? null : this._symbols[code];
|
|
1514
2470
|
}
|
|
2471
|
+
/**
|
|
2472
|
+
* The text of one row: the text Qlik displays for its value.
|
|
2473
|
+
*
|
|
2474
|
+
* A string is its own text and a dual has its own. A value read as one half of a symbol - a date
|
|
2475
|
+
* read as its serial, a string read as a number - has the text the file stores for it, and a pure
|
|
2476
|
+
* number has none.
|
|
2477
|
+
*
|
|
2478
|
+
* @param {number} row The row index.
|
|
2479
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
2480
|
+
* @throws {QvdValidationError} If the row is not an integer within the column.
|
|
2481
|
+
*/
|
|
2482
|
+
textAt(row) {
|
|
2483
|
+
if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
|
|
2484
|
+
throw new QvdValidationError("Row index out of bounds", {
|
|
2485
|
+
column: this._name,
|
|
2486
|
+
row,
|
|
2487
|
+
length: this._codes.length
|
|
2488
|
+
});
|
|
2489
|
+
}
|
|
2490
|
+
const code = this._codes[row];
|
|
2491
|
+
if (code < 0) {
|
|
2492
|
+
return null;
|
|
2493
|
+
}
|
|
2494
|
+
return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
|
|
2495
|
+
}
|
|
2496
|
+
/**
|
|
2497
|
+
* The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
|
|
2498
|
+
*
|
|
2499
|
+
* @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
|
|
2500
|
+
*/
|
|
2501
|
+
symbolTexts() {
|
|
2502
|
+
if (this._halves !== null) {
|
|
2503
|
+
return this._halves.texts;
|
|
2504
|
+
}
|
|
2505
|
+
return Object.freeze(this._symbols.map(textOf));
|
|
2506
|
+
}
|
|
1515
2507
|
/**
|
|
1516
2508
|
* Iterates the column's values without materialising it.
|
|
1517
2509
|
*
|
|
@@ -1555,6 +2547,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1555
2547
|
* one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
|
|
1556
2548
|
* codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
|
|
1557
2549
|
*
|
|
2550
|
+
* A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
|
|
2551
|
+
* `{duals: 'text'}` gives the serial the file stores for it, not NaN.
|
|
2552
|
+
*
|
|
1558
2553
|
* Scanning `codes` against this is the fastest way to read a column, because both sides are
|
|
1559
2554
|
* contiguous typed arrays and the dictionary fits in cache:
|
|
1560
2555
|
*
|
|
@@ -1576,8 +2571,7 @@ var init_QvdColumnTable = __esm({
|
|
|
1576
2571
|
numericSymbols() {
|
|
1577
2572
|
const out = new Float64Array(this._symbols.length);
|
|
1578
2573
|
for (let index = 0; index < this._symbols.length; index++) {
|
|
1579
|
-
|
|
1580
|
-
out[index] = typeof value === "number" ? value : NaN;
|
|
2574
|
+
out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
|
|
1581
2575
|
}
|
|
1582
2576
|
return out;
|
|
1583
2577
|
}
|
|
@@ -1592,7 +2586,8 @@ var init_QvdColumnTable = __esm({
|
|
|
1592
2586
|
* @param {Object} [options] Conversion options.
|
|
1593
2587
|
* @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
|
|
1594
2588
|
* number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
|
|
1595
|
-
* NaN, which is the right choice only when the caller knows the column is numeric.
|
|
2589
|
+
* NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
|
|
2590
|
+
* number here, as it is to `numericSymbols`.
|
|
1596
2591
|
* @return {Float64Array} One number per row.
|
|
1597
2592
|
* @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
|
|
1598
2593
|
*/
|
|
@@ -1612,6 +2607,11 @@ var init_QvdColumnTable = __esm({
|
|
|
1612
2607
|
out[row] = value;
|
|
1613
2608
|
continue;
|
|
1614
2609
|
}
|
|
2610
|
+
const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
|
|
2611
|
+
if (!Number.isNaN(number)) {
|
|
2612
|
+
out[row] = number;
|
|
2613
|
+
continue;
|
|
2614
|
+
}
|
|
1615
2615
|
if (onNonNumeric === "throw") {
|
|
1616
2616
|
throw new QvdValidationError("Column holds a value that is not a number", {
|
|
1617
2617
|
column: this._name,
|
|
@@ -1627,29 +2627,57 @@ var init_QvdColumnTable = __esm({
|
|
|
1627
2627
|
}
|
|
1628
2628
|
};
|
|
1629
2629
|
QvdColumnTable = class {
|
|
2630
|
+
static {
|
|
2631
|
+
__name(this, "QvdColumnTable");
|
|
2632
|
+
}
|
|
1630
2633
|
/**
|
|
1631
2634
|
* @param {Object} decoded What the reader decoded.
|
|
1632
2635
|
* @param {Array<string>} decoded.columns Field names, in file order.
|
|
1633
2636
|
* @param {Array<Int32Array>} decoded.codesByField One code array per field.
|
|
1634
2637
|
* @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
|
|
2638
|
+
* @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
|
|
2639
|
+
* or null for a field whose values show them.
|
|
1635
2640
|
* @param {number} decoded.rowCount Rows decoded.
|
|
1636
2641
|
* @param {any} decoded.metadata The raw QvdTableHeader.
|
|
2642
|
+
* @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
|
|
2643
|
+
* stored-symbol record, as a data frame of the same read carries it.
|
|
1637
2644
|
* @param {any} decoded.loadStats Statistics about the read.
|
|
1638
2645
|
*/
|
|
1639
|
-
constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
|
|
2646
|
+
constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
|
|
1640
2647
|
this._columns = columns;
|
|
1641
2648
|
this._codesByField = codesByField;
|
|
1642
2649
|
this._symbolsByField = symbolsByField;
|
|
2650
|
+
this._halvesByField = halvesByField ?? null;
|
|
1643
2651
|
this._rowCount = rowCount;
|
|
1644
2652
|
this._metadata = metadata;
|
|
2653
|
+
this._storedSymbols = storedSymbols ?? null;
|
|
1645
2654
|
this._loadStats = loadStats;
|
|
1646
2655
|
}
|
|
1647
2656
|
/**
|
|
1648
2657
|
* Reads a QVD file as columns.
|
|
1649
2658
|
*
|
|
2659
|
+
* Takes the same options as `QvdDataFrame.fromQvd`, with the same meanings - one option
|
|
2660
|
+
* vocabulary for both read paths, because they are two answers about the same file rather than
|
|
2661
|
+
* two features. `{offset, limit}` is how a caller pages through a file columnwise; there is no
|
|
2662
|
+
* columnar `iterate()` because there is nothing for it to bound - a columnar read materialises
|
|
2663
|
+
* no rows, which is the memory chunking exists to cap.
|
|
2664
|
+
*
|
|
1650
2665
|
* @param {string} path The path to the QVD file.
|
|
1651
2666
|
* @param {Object} [options] Loading options, with the same meanings they have on `fromQvd`.
|
|
1652
|
-
* @param {number|null} [options.maxRows]
|
|
2667
|
+
* @param {number|null} [options.maxRows] Rows to decode. The older name for `limit`.
|
|
2668
|
+
* @param {number|null} [options.limit] Rows to decode, counting from `offset`.
|
|
2669
|
+
* @param {number} [options.offset] File row to start at.
|
|
2670
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
2671
|
+
* appear. Unselected fields have their symbols skipped entirely.
|
|
2672
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
|
|
2673
|
+
* number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
|
|
2674
|
+
* text and `numericSymbols` the number. Anything else throws.
|
|
2675
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
|
|
2676
|
+
* number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
2677
|
+
* `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
|
|
2678
|
+
* Anything but a boolean throws.
|
|
2679
|
+
* @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
|
|
2680
|
+
* @param {AbortSignal} [options.signal] Cancels the read.
|
|
1653
2681
|
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
1654
2682
|
* @param {number} [options.memorySafetyFactor] Fraction of the memory budget a load may use.
|
|
1655
2683
|
* @param {number} [options.symbolFilteringThreshold] Symbol table size above which a limited
|
|
@@ -1659,15 +2687,13 @@ var init_QvdColumnTable = __esm({
|
|
|
1659
2687
|
static async fromQvd(path3, options = {}) {
|
|
1660
2688
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
1661
2689
|
const reader = new QvdFileReader2(path3, {
|
|
1662
|
-
|
|
1663
|
-
memorySafetyFactor: options.memorySafetyFactor,
|
|
1664
|
-
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
2690
|
+
...readerOptionsFrom(options),
|
|
1665
2691
|
// This read builds no rows, so the memory guard must not charge it for them. A columnar
|
|
1666
2692
|
// read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
|
|
1667
2693
|
// refused below a 2GB one.
|
|
1668
2694
|
materialisesRows: false
|
|
1669
2695
|
});
|
|
1670
|
-
return await reader.loadColumnar(options
|
|
2696
|
+
return await reader.loadColumnar(windowFrom(options));
|
|
1671
2697
|
}
|
|
1672
2698
|
/** @return {Array<string>} Field names, in file order. */
|
|
1673
2699
|
get columns() {
|
|
@@ -1689,6 +2715,14 @@ var init_QvdColumnTable = __esm({
|
|
|
1689
2715
|
get loadStats() {
|
|
1690
2716
|
return this._loadStats;
|
|
1691
2717
|
}
|
|
2718
|
+
/**
|
|
2719
|
+
* The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
|
|
2720
|
+
*
|
|
2721
|
+
* @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
|
|
2722
|
+
*/
|
|
2723
|
+
get storedSymbols() {
|
|
2724
|
+
return this._storedSymbols;
|
|
2725
|
+
}
|
|
1692
2726
|
/**
|
|
1693
2727
|
* One column.
|
|
1694
2728
|
*
|
|
@@ -1704,7 +2738,12 @@ var init_QvdColumnTable = __esm({
|
|
|
1704
2738
|
availableColumns: this._columns
|
|
1705
2739
|
});
|
|
1706
2740
|
}
|
|
1707
|
-
return new QvdColumn(
|
|
2741
|
+
return new QvdColumn(
|
|
2742
|
+
name,
|
|
2743
|
+
this._codesByField[index],
|
|
2744
|
+
this._symbolsByField[index],
|
|
2745
|
+
this._halvesByField?.[index] ?? null
|
|
2746
|
+
);
|
|
1708
2747
|
}
|
|
1709
2748
|
};
|
|
1710
2749
|
}
|
|
@@ -1715,7 +2754,17 @@ var QvdFileReader_exports = {};
|
|
|
1715
2754
|
__export(QvdFileReader_exports, {
|
|
1716
2755
|
QvdFileReader: () => QvdFileReader
|
|
1717
2756
|
});
|
|
1718
|
-
|
|
2757
|
+
function closeReadStream(stream) {
|
|
2758
|
+
if (stream.closed) {
|
|
2759
|
+
return Promise.resolve();
|
|
2760
|
+
}
|
|
2761
|
+
return new Promise((resolve) => {
|
|
2762
|
+
stream.once("close", () => resolve());
|
|
2763
|
+
stream.once("error", () => resolve());
|
|
2764
|
+
stream.destroy();
|
|
2765
|
+
});
|
|
2766
|
+
}
|
|
2767
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
|
|
1719
2768
|
var init_QvdFileReader = __esm({
|
|
1720
2769
|
"src/QvdFileReader.js"() {
|
|
1721
2770
|
init_QvdDataFrame();
|
|
@@ -1725,9 +2774,17 @@ var init_QvdFileReader = __esm({
|
|
|
1725
2774
|
init_memoryUtils();
|
|
1726
2775
|
init_validationUtils();
|
|
1727
2776
|
init_symbolParser();
|
|
2777
|
+
init_readOptions();
|
|
2778
|
+
init_resolveSymbols();
|
|
2779
|
+
init_storedSymbols();
|
|
1728
2780
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
1729
2781
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
2782
|
+
ANALYSIS_SLICE_ROWS = 65536;
|
|
2783
|
+
__name(closeReadStream, "closeReadStream");
|
|
1730
2784
|
QvdFileReader = class {
|
|
2785
|
+
static {
|
|
2786
|
+
__name(this, "QvdFileReader");
|
|
2787
|
+
}
|
|
1731
2788
|
/**
|
|
1732
2789
|
* Constructs a new QVD file parser.
|
|
1733
2790
|
*
|
|
@@ -1738,9 +2795,10 @@ var init_QvdFileReader = __esm({
|
|
|
1738
2795
|
* points outside it is rejected. Defaults to the current working directory. To permit
|
|
1739
2796
|
* an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
|
|
1740
2797
|
* empty value falls back to the working directory rather than removing the restriction.
|
|
1741
|
-
* @param {number} [options.memorySafetyFactor=0.
|
|
1742
|
-
* load may use. The budget is the
|
|
1743
|
-
*
|
|
2798
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction (0.0-1.0) of the memory budget a
|
|
2799
|
+
* load may use. The budget is the smaller of the V8 heap limit and any container memory limit;
|
|
2800
|
+
* what the OS reports as available is recorded for diagnostics and deliberately not allowed to
|
|
2801
|
+
* bind - see `getMemoryBudget`. Default is 0.8. **Zero disables the memory
|
|
1744
2802
|
* check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
|
|
1745
2803
|
* Bun reports its current heap as its heap limit - and for callers who would rather manage
|
|
1746
2804
|
* memory themselves than trust the estimate.
|
|
@@ -1751,41 +2809,123 @@ var init_QvdFileReader = __esm({
|
|
|
1751
2809
|
* above which a lazy load switches to the two-pass filtering path. The default of 50MB is
|
|
1752
2810
|
* the point where the extra analysis pass pays for itself; lower it to use filtering on
|
|
1753
2811
|
* smaller files, raise it to keep the simpler single-pass read for longer.
|
|
2812
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
2813
|
+
* appear. Null reads every field, in file order. An unknown or repeated name is refused.
|
|
2814
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
2815
|
+
* text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
|
|
2816
|
+
* value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
|
|
2817
|
+
* `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
|
|
2818
|
+
* `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
|
|
2819
|
+
* the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
|
|
2820
|
+
* throws a `QvdValidationError`.
|
|
2821
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
2822
|
+
* reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
|
|
2823
|
+
* every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
|
|
2824
|
+
* dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
|
|
2825
|
+
* dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
|
|
2826
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
2827
|
+
* read proceeds - the same shape `QvdFileWriter` emits.
|
|
2828
|
+
* @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
|
|
2829
|
+
* `signal.reason`, exactly as `signal.throwIfAborted()` does.
|
|
1754
2830
|
*/
|
|
1755
2831
|
constructor(filePath, options = {}) {
|
|
1756
2832
|
const {
|
|
1757
2833
|
allowedDir,
|
|
1758
2834
|
memorySafetyFactor = 0.8,
|
|
1759
2835
|
symbolFilteringThreshold = 50 * 1024 * 1024,
|
|
1760
|
-
materialisesRows = true
|
|
2836
|
+
materialisesRows = true,
|
|
2837
|
+
fields = null,
|
|
2838
|
+
duals,
|
|
2839
|
+
coerceNumericStrings,
|
|
2840
|
+
onProgress,
|
|
2841
|
+
signal
|
|
1761
2842
|
} = options;
|
|
1762
2843
|
this._materialisesRows = materialisesRows;
|
|
1763
2844
|
this._path = validatePath(filePath, allowedDir);
|
|
2845
|
+
this._duals = normaliseDuals(duals, this._path);
|
|
2846
|
+
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
1764
2847
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
1765
2848
|
this._symbolFilteringThreshold = symbolFilteringThreshold;
|
|
2849
|
+
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
2850
|
+
throw new QvdValidationError("onProgress must be a function", {
|
|
2851
|
+
provided: onProgress,
|
|
2852
|
+
type: typeof onProgress,
|
|
2853
|
+
file: this._path
|
|
2854
|
+
});
|
|
2855
|
+
}
|
|
2856
|
+
if (signal !== void 0 && (typeof signal !== "object" || signal === null || typeof signal.aborted !== "boolean")) {
|
|
2857
|
+
throw new QvdValidationError("signal must be an AbortSignal", {
|
|
2858
|
+
provided: signal,
|
|
2859
|
+
type: typeof signal,
|
|
2860
|
+
file: this._path
|
|
2861
|
+
});
|
|
2862
|
+
}
|
|
2863
|
+
this._requestedFields = fields === void 0 ? null : fields;
|
|
2864
|
+
this._onProgress = onProgress;
|
|
2865
|
+
this._signal = signal;
|
|
1766
2866
|
this._buffer = null;
|
|
1767
2867
|
this._headerOffset = null;
|
|
1768
2868
|
this._symbolTableOffset = null;
|
|
1769
2869
|
this._indexTableOffset = null;
|
|
1770
2870
|
this._header = null;
|
|
2871
|
+
this._allFields = null;
|
|
2872
|
+
this._selectedFields = null;
|
|
2873
|
+
this._fieldBitMetadataValidated = false;
|
|
1771
2874
|
this._symbolTable = null;
|
|
1772
2875
|
this._indexColumns = null;
|
|
1773
2876
|
this._rowsDecoded = 0;
|
|
2877
|
+
this._bufferFirstRow = 0;
|
|
1774
2878
|
this._fileSize = null;
|
|
1775
2879
|
this._headerMatchesFile = false;
|
|
1776
2880
|
}
|
|
2881
|
+
/**
|
|
2882
|
+
* Emits a progress event if a callback is registered.
|
|
2883
|
+
*
|
|
2884
|
+
* The same shape `QvdFileWriter._emitProgress` emits, deliberately: a caller who has written a
|
|
2885
|
+
* progress bar for a write should not have to write a second one for a read. The stage names
|
|
2886
|
+
* differ because the stages differ, but `symbol-table` and `index-table` mean the same thing on
|
|
2887
|
+
* both sides.
|
|
2888
|
+
*
|
|
2889
|
+
* @param {string} stage The current stage of the read.
|
|
2890
|
+
* @param {number} current The current progress value.
|
|
2891
|
+
* @param {number} total The total progress value.
|
|
2892
|
+
* @private
|
|
2893
|
+
*/
|
|
2894
|
+
_emitProgress(stage, current, total) {
|
|
2895
|
+
if (this._onProgress) {
|
|
2896
|
+
const percent = total > 0 ? Math.round(current / total * 100) : 100;
|
|
2897
|
+
this._onProgress({ stage, current, total, percent });
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
/**
|
|
2901
|
+
* Throws if the caller has cancelled the read.
|
|
2902
|
+
*
|
|
2903
|
+
* Throws `signal.reason` - a `DOMException` named `AbortError` unless the caller aborted with a
|
|
2904
|
+
* reason of their own. That is what `AbortSignal` means everywhere else in Node, and inventing
|
|
2905
|
+
* a `QvdAbortError` here would make this library's cancellation the one a caller has to special
|
|
2906
|
+
* case.
|
|
2907
|
+
*
|
|
2908
|
+
* @private
|
|
2909
|
+
*/
|
|
2910
|
+
_throwIfAborted() {
|
|
2911
|
+
if (this._signal) {
|
|
2912
|
+
this._signal.throwIfAborted();
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
1777
2915
|
/**
|
|
1778
2916
|
* Reads the binary data of the QVD file.
|
|
1779
2917
|
*
|
|
1780
|
-
*
|
|
1781
|
-
*
|
|
2918
|
+
* A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
|
|
2919
|
+
* needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
|
|
2920
|
+
* over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
|
|
1782
2921
|
*
|
|
1783
|
-
*
|
|
1784
|
-
*
|
|
1785
|
-
*
|
|
1786
|
-
*
|
|
2922
|
+
* The saving is in the index table and the rows, not in the symbol table, which is read in
|
|
2923
|
+
* full whatever the window because a stored index in any row can address any symbol. So the
|
|
2924
|
+
* gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
|
|
2925
|
+
* values there is very little to save, which is what `symbolFilteringThreshold` and the
|
|
2926
|
+
* two-pass path exist for.
|
|
1787
2927
|
*
|
|
1788
|
-
* Algorithm for
|
|
2928
|
+
* Algorithm for a windowed read:
|
|
1789
2929
|
* 1. Stream-read the file until XML header delimiter is found
|
|
1790
2930
|
* 2. Parse header to determine symbol table and index table locations
|
|
1791
2931
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
@@ -1798,14 +2938,23 @@ var init_QvdFileReader = __esm({
|
|
|
1798
2938
|
* - Streaming for header finding is efficient for unknown header sizes
|
|
1799
2939
|
* - Direct byte-range reading for remaining data is fastest
|
|
1800
2940
|
*
|
|
1801
|
-
*
|
|
2941
|
+
* A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
|
|
2942
|
+
* table from the front of the file, and the window's records from wherever they sit. The bytes
|
|
2943
|
+
* between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
|
|
2944
|
+
* fixture a 0.4MB read rather than a 38MB one.
|
|
2945
|
+
*
|
|
2946
|
+
* @param {QvdRowWindow} window The rows to read.
|
|
1802
2947
|
* @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
|
|
1803
2948
|
* symbol and index tables on disk. This is the metadata-only path: the header is a few
|
|
1804
2949
|
* kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
|
|
1805
2950
|
* for a 40GB one.
|
|
2951
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
|
|
2952
|
+
* that is fewer than the window covers - see `_prepare`.
|
|
1806
2953
|
* @private
|
|
1807
2954
|
*/
|
|
1808
|
-
async _readData(
|
|
2955
|
+
async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
|
|
2956
|
+
this._throwIfAborted();
|
|
2957
|
+
this._emitProgress("read", 0, 1);
|
|
1809
2958
|
const HEADER_DELIMITER = "\r\n\0";
|
|
1810
2959
|
const CHUNK_SIZE = 64 * 1024;
|
|
1811
2960
|
const stream = fs.createReadStream(this._path, {
|
|
@@ -1847,6 +2996,8 @@ var init_QvdFileReader = __esm({
|
|
|
1847
2996
|
if (!isExpectedEarlyClose) {
|
|
1848
2997
|
throw error;
|
|
1849
2998
|
}
|
|
2999
|
+
} finally {
|
|
3000
|
+
await closeReadStream(stream);
|
|
1850
3001
|
}
|
|
1851
3002
|
if (headerDelimiterIndex === -1) {
|
|
1852
3003
|
throw new QvdCorruptedError(
|
|
@@ -1875,13 +3026,14 @@ var init_QvdFileReader = __esm({
|
|
|
1875
3026
|
const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
|
|
1876
3027
|
if (headerOnly) {
|
|
1877
3028
|
this._buffer = headerBuffer.subarray(0, headerEndIndex);
|
|
3029
|
+
this._emitProgress("read", 1, 1);
|
|
1878
3030
|
return;
|
|
1879
3031
|
}
|
|
1880
3032
|
let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
|
|
1881
3033
|
if (headerFields && !Array.isArray(headerFields)) {
|
|
1882
3034
|
headerFields = [headerFields];
|
|
1883
3035
|
}
|
|
1884
|
-
const columnCount = Array.isArray(headerFields) ? headerFields.length : 0;
|
|
3036
|
+
const columnCount = Array.isArray(headerFields) ? selectFields(headerFields, this._requestedFields, this._path).length : 0;
|
|
1885
3037
|
const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
|
|
1886
3038
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
1887
3039
|
);
|
|
@@ -1890,23 +3042,28 @@ var init_QvdFileReader = __esm({
|
|
|
1890
3042
|
this._fileSize = fileSize;
|
|
1891
3043
|
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
|
|
1892
3044
|
}
|
|
3045
|
+
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3046
|
+
const windowRows = resolved.limit;
|
|
1893
3047
|
if (headerNumbersUsable && this._headerMatchesFile) {
|
|
1894
3048
|
validateMemoryAvailability(
|
|
1895
3049
|
symbolTableLength,
|
|
1896
|
-
|
|
3050
|
+
windowRows,
|
|
1897
3051
|
totalRows,
|
|
1898
3052
|
this._path,
|
|
1899
3053
|
this._memorySafetyFactor,
|
|
1900
3054
|
columnCount,
|
|
1901
|
-
this._materialisesRows
|
|
3055
|
+
this._materialisesRows,
|
|
3056
|
+
liveRows
|
|
1902
3057
|
);
|
|
1903
3058
|
}
|
|
1904
|
-
if (
|
|
3059
|
+
if (window.offset === 0 && window.limit === null) {
|
|
1905
3060
|
this._buffer = await fs.promises.readFile(this._path);
|
|
1906
3061
|
this._fileSize = this._buffer.length;
|
|
3062
|
+
this._bufferFirstRow = 0;
|
|
3063
|
+
this._emitProgress("read", 1, 1);
|
|
1907
3064
|
return;
|
|
1908
3065
|
}
|
|
1909
|
-
const rowsToLoad =
|
|
3066
|
+
const rowsToLoad = windowRows;
|
|
1910
3067
|
validateSymbolTableSizeEarly(symbolTableLength, this._path);
|
|
1911
3068
|
for (const [name, value] of [
|
|
1912
3069
|
["Offset", symbolTableLength],
|
|
@@ -1922,39 +3079,78 @@ var init_QvdFileReader = __esm({
|
|
|
1922
3079
|
});
|
|
1923
3080
|
}
|
|
1924
3081
|
}
|
|
3082
|
+
const skippedIndexBytes = resolved.offset * recordSize;
|
|
1925
3083
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
1926
3084
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
3085
|
+
const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
|
|
1927
3086
|
const fd = await fs.promises.open(this._path, "r");
|
|
1928
3087
|
try {
|
|
1929
3088
|
const { size: fileSize } = await fd.stat();
|
|
1930
3089
|
this._fileSize = fileSize;
|
|
1931
|
-
if (
|
|
3090
|
+
if (fileBytesRequired > fileSize) {
|
|
1932
3091
|
throw new QvdCorruptedError("The file is shorter than its header claims.", {
|
|
1933
3092
|
file: this._path,
|
|
1934
3093
|
fileSize,
|
|
1935
|
-
requiredBytes:
|
|
3094
|
+
requiredBytes: fileBytesRequired,
|
|
1936
3095
|
stage: "readData"
|
|
1937
3096
|
});
|
|
1938
3097
|
}
|
|
1939
3098
|
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
stage: "readData"
|
|
1951
|
-
});
|
|
1952
|
-
}
|
|
1953
|
-
position += bytesRead;
|
|
3099
|
+
await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3100
|
+
if (indexTableBytesToRead > 0) {
|
|
3101
|
+
await this._readRange(
|
|
3102
|
+
fd,
|
|
3103
|
+
indexTableOffset,
|
|
3104
|
+
indexTableBytesToRead,
|
|
3105
|
+
indexTableOffset + skippedIndexBytes,
|
|
3106
|
+
fileSize,
|
|
3107
|
+
fileBytesRequired
|
|
3108
|
+
);
|
|
1954
3109
|
}
|
|
3110
|
+
this._bufferFirstRow = resolved.offset;
|
|
1955
3111
|
} finally {
|
|
1956
3112
|
await fd.close();
|
|
1957
3113
|
}
|
|
3114
|
+
this._emitProgress("read", 1, 1);
|
|
3115
|
+
}
|
|
3116
|
+
/**
|
|
3117
|
+
* Reads one byte range of the file into the buffer.
|
|
3118
|
+
*
|
|
3119
|
+
* Read in bounded chunks, checking bytesRead each time. A single fs.read call with a length of
|
|
3120
|
+
* 2^31 or more does not throw - it trips a C++ assertion and aborts the whole process, which no
|
|
3121
|
+
* try/catch can intercept.
|
|
3122
|
+
*
|
|
3123
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
3124
|
+
* @param {number} bufferOffset Where in the buffer to write.
|
|
3125
|
+
* @param {number} byteCount How many bytes to read.
|
|
3126
|
+
* @param {number} filePosition Where in the file to read from.
|
|
3127
|
+
* @param {number} fileSize The file's size, for the error.
|
|
3128
|
+
* @param {number} requiredBytes Bytes the whole read needs, for the error.
|
|
3129
|
+
* @private
|
|
3130
|
+
*/
|
|
3131
|
+
async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
|
|
3132
|
+
assert2(this._buffer, "The read buffer has not been allocated.");
|
|
3133
|
+
let done = 0;
|
|
3134
|
+
while (done < byteCount) {
|
|
3135
|
+
const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
|
|
3136
|
+
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
|
|
3137
|
+
if (bytesRead === 0) {
|
|
3138
|
+
throw new QvdCorruptedError("Unexpected end of file while reading QVD data.", {
|
|
3139
|
+
file: this._path,
|
|
3140
|
+
fileSize,
|
|
3141
|
+
// Two numbers, because they stopped being the same one when a window began reading two
|
|
3142
|
+
// ranges: `bytesRead` is how much of this range arrived, `filePosition` is where in the
|
|
3143
|
+
// file it gave up. Reporting the position under the name of the count made a windowed
|
|
3144
|
+
// read of a truncated file claim tens of megabytes had been read when a few hundred
|
|
3145
|
+
// bytes had.
|
|
3146
|
+
bytesRead: done,
|
|
3147
|
+
filePosition: filePosition + done,
|
|
3148
|
+
requiredBytes,
|
|
3149
|
+
stage: "readData"
|
|
3150
|
+
});
|
|
3151
|
+
}
|
|
3152
|
+
done += bytesRead;
|
|
3153
|
+
}
|
|
1958
3154
|
}
|
|
1959
3155
|
/**
|
|
1960
3156
|
* Parses the XML header of the QVD file. This method is part of the parsing process
|
|
@@ -2014,6 +3210,8 @@ var init_QvdFileReader = __esm({
|
|
|
2014
3210
|
this._headerOffset = headerBeginIndex;
|
|
2015
3211
|
this._symbolTableOffset = headerEndIndex;
|
|
2016
3212
|
this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3213
|
+
this._allFields = fieldList;
|
|
3214
|
+
this._selectedFields = selectFields(this._allFields, this._requestedFields, this._path);
|
|
2017
3215
|
}
|
|
2018
3216
|
/**
|
|
2019
3217
|
* Establishes the geometry of the index table, and validates it.
|
|
@@ -2024,30 +3222,29 @@ var init_QvdFileReader = __esm({
|
|
|
2024
3222
|
* about keeping the sign in step with the other one: the two could drift, and #113 is what
|
|
2025
3223
|
* that looks like when they do. There is one copy now.
|
|
2026
3224
|
*
|
|
2027
|
-
* @param {
|
|
3225
|
+
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
2028
3226
|
* @param {string} stage Stage name for any error raised here.
|
|
2029
3227
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
2030
|
-
* indexBuffer: Buffer}} The record geometry.
|
|
3228
|
+
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the window's first
|
|
3229
|
+
* record, so the decoder always counts from zero.
|
|
2031
3230
|
* @private
|
|
2032
3231
|
*/
|
|
2033
|
-
_planIndexTable(
|
|
2034
|
-
if (!this._buffer || !this._header || !this._indexTableOffset) {
|
|
3232
|
+
_planIndexTable(window, stage) {
|
|
3233
|
+
if (!this._buffer || !this._header || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
2035
3234
|
throw new QvdCorruptedError(
|
|
2036
3235
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
2037
3236
|
{
|
|
2038
3237
|
file: this._path,
|
|
2039
3238
|
stage
|
|
2040
3239
|
}
|
|
2041
|
-
);
|
|
2042
|
-
}
|
|
2043
|
-
let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
|
|
2044
|
-
if (!Array.isArray(fields)) {
|
|
2045
|
-
fields = [fields];
|
|
3240
|
+
);
|
|
2046
3241
|
}
|
|
3242
|
+
const allFields = this._allFields;
|
|
3243
|
+
const fields = this._selectedFields;
|
|
2047
3244
|
const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
|
|
2048
3245
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
2049
|
-
const rowsToLoad = rowLimit !== null ? Math.min(rowLimit, totalRows) : totalRows;
|
|
2050
3246
|
const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
|
|
3247
|
+
const { offset: firstRow, limit: rowsToLoad } = resolveWindow(window, totalRows);
|
|
2051
3248
|
validateIndexTableMetadata(
|
|
2052
3249
|
recordSize,
|
|
2053
3250
|
totalRows,
|
|
@@ -2056,11 +3253,20 @@ var init_QvdFileReader = __esm({
|
|
|
2056
3253
|
this._buffer.length,
|
|
2057
3254
|
rowsToLoad,
|
|
2058
3255
|
this._path,
|
|
2059
|
-
this._fileSize
|
|
3256
|
+
this._fileSize,
|
|
3257
|
+
firstRow,
|
|
3258
|
+
this._bufferFirstRow
|
|
3259
|
+
);
|
|
3260
|
+
const bufferRecordStart = (firstRow - this._bufferFirstRow) * recordSize;
|
|
3261
|
+
const indexBuffer = this._buffer.subarray(
|
|
3262
|
+
this._indexTableOffset + bufferRecordStart,
|
|
3263
|
+
this._indexTableOffset + bufferRecordStart + rowsToLoad * recordSize
|
|
2060
3264
|
);
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
3265
|
+
if (!this._fieldBitMetadataValidated) {
|
|
3266
|
+
for (const field of allFields) {
|
|
3267
|
+
validateFieldBitMetadata(field, recordSize, this._path);
|
|
3268
|
+
}
|
|
3269
|
+
this._fieldBitMetadataValidated = true;
|
|
2064
3270
|
}
|
|
2065
3271
|
assert2(
|
|
2066
3272
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
@@ -2072,31 +3278,45 @@ var init_QvdFileReader = __esm({
|
|
|
2072
3278
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
2073
3279
|
* This is used for two-pass symbol filtering optimization.
|
|
2074
3280
|
*
|
|
2075
|
-
*
|
|
2076
|
-
*
|
|
3281
|
+
* Only the selected fields are analysed. An unselected field's symbols are never parsed, so
|
|
3282
|
+
* there is nothing for a usage set to filter and decoding its column would be a pass over the
|
|
3283
|
+
* whole window for an answer nobody reads.
|
|
3284
|
+
*
|
|
3285
|
+
* @param {QvdRowWindow} window The rows to analyse.
|
|
3286
|
+
* @return {Promise<Array<Set<number>>>} One set of needed symbol indices per selected field, in
|
|
3287
|
+
* the same order `_parseSymbolTable` walks them.
|
|
2077
3288
|
* @private
|
|
2078
3289
|
*/
|
|
2079
|
-
async _analyzeIndexTableSymbolUsage(
|
|
2080
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(
|
|
2081
|
-
const symbolUsage =
|
|
2082
|
-
const
|
|
2083
|
-
|
|
3290
|
+
async _analyzeIndexTableSymbolUsage(window) {
|
|
3291
|
+
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "analyzeIndexTableSymbolUsage");
|
|
3292
|
+
const symbolUsage = [];
|
|
3293
|
+
const sliceRows = Math.min(rowsToLoad, ANALYSIS_SLICE_ROWS);
|
|
3294
|
+
const column = new Int32Array(sliceRows);
|
|
3295
|
+
fields.forEach((field, position) => {
|
|
3296
|
+
this._throwIfAborted();
|
|
2084
3297
|
const needed = /* @__PURE__ */ new Set();
|
|
2085
|
-
symbolUsage
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
3298
|
+
symbolUsage[position] = needed;
|
|
3299
|
+
const bitOffset = parseInt(field["BitOffset"], 10);
|
|
3300
|
+
const bitWidth = parseInt(field["BitWidth"], 10);
|
|
3301
|
+
const bias = parseInt(field["Bias"], 10);
|
|
3302
|
+
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3303
|
+
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3304
|
+
decodeIndexColumn(
|
|
3305
|
+
first === 0 ? indexBuffer : indexBuffer.subarray(first * recordSize),
|
|
3306
|
+
recordSize,
|
|
3307
|
+
count,
|
|
3308
|
+
bitOffset,
|
|
3309
|
+
bitWidth,
|
|
3310
|
+
bias,
|
|
3311
|
+
column
|
|
3312
|
+
);
|
|
3313
|
+
for (let row = 0; row < count; row++) {
|
|
3314
|
+
if (column[row] >= 0) {
|
|
3315
|
+
needed.add(column[row]);
|
|
3316
|
+
}
|
|
2098
3317
|
}
|
|
2099
3318
|
}
|
|
3319
|
+
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
2100
3320
|
});
|
|
2101
3321
|
return symbolUsage;
|
|
2102
3322
|
}
|
|
@@ -2104,12 +3324,20 @@ var init_QvdFileReader = __esm({
|
|
|
2104
3324
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
2105
3325
|
* and should not be called directly.
|
|
2106
3326
|
*
|
|
2107
|
-
*
|
|
2108
|
-
*
|
|
2109
|
-
*
|
|
3327
|
+
* A field the caller did not select is skipped whole. Its symbol area is neither scanned nor
|
|
3328
|
+
* parsed - the per-field `Offset` and `Length` say exactly where it is, so there is nothing to
|
|
3329
|
+
* walk past - and that is where field selection earns its keep. The index decode is cheap by
|
|
3330
|
+
* comparison; parsing symbols is not.
|
|
3331
|
+
*
|
|
3332
|
+
* @param {Array<Set<number>>|null} symbolsToKeep Optional set of symbol indices to keep per
|
|
3333
|
+
* selected field, indexed by position. If provided, only these symbols will be parsed
|
|
3334
|
+
* (two-pass filtering optimization).
|
|
3335
|
+
* @param {number} rowsToLoad Rows the read covers, for memory estimation.
|
|
3336
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows=null] Rows held at one instant when
|
|
3337
|
+
* that is fewer than the window covers - see `_prepare`.
|
|
2110
3338
|
*/
|
|
2111
|
-
async _parseSymbolTable(symbolsToKeep = null,
|
|
2112
|
-
if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset) {
|
|
3339
|
+
async _parseSymbolTable(symbolsToKeep = null, rowsToLoad = 0, liveRows = null) {
|
|
3340
|
+
if (!this._buffer || !this._header || !this._symbolTableOffset || !this._indexTableOffset || !this._selectedFields || !this._allFields) {
|
|
2113
3341
|
throw new QvdCorruptedError(
|
|
2114
3342
|
"The QVD file has not been loaded in the proper order or has not been loaded at all.",
|
|
2115
3343
|
{
|
|
@@ -2118,7 +3346,8 @@ var init_QvdFileReader = __esm({
|
|
|
2118
3346
|
}
|
|
2119
3347
|
);
|
|
2120
3348
|
}
|
|
2121
|
-
|
|
3349
|
+
const allFields = this._allFields;
|
|
3350
|
+
const fields = this._selectedFields;
|
|
2122
3351
|
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
2123
3352
|
const symbolTableSize = symbolBuffer.length;
|
|
2124
3353
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
@@ -2126,51 +3355,34 @@ var init_QvdFileReader = __esm({
|
|
|
2126
3355
|
if (this._headerMatchesFile) {
|
|
2127
3356
|
validateMemoryAvailability(
|
|
2128
3357
|
symbolTableSize,
|
|
2129
|
-
|
|
3358
|
+
rowsToLoad,
|
|
2130
3359
|
totalRows,
|
|
2131
3360
|
this._path,
|
|
2132
3361
|
this._memorySafetyFactor,
|
|
2133
|
-
|
|
2134
|
-
this._materialisesRows
|
|
3362
|
+
fields.length,
|
|
3363
|
+
this._materialisesRows,
|
|
3364
|
+
liveRows
|
|
2135
3365
|
);
|
|
2136
3366
|
}
|
|
2137
|
-
warnLargeSymbolTable(
|
|
2138
|
-
|
|
2139
|
-
maxRows,
|
|
2140
|
-
totalRows,
|
|
2141
|
-
Array.isArray(fields) ? fields.length : 1,
|
|
2142
|
-
this._materialisesRows
|
|
2143
|
-
);
|
|
2144
|
-
if (!Array.isArray(fields)) {
|
|
2145
|
-
fields = [fields];
|
|
2146
|
-
}
|
|
2147
|
-
for (const field of fields) {
|
|
3367
|
+
warnLargeSymbolTable(symbolTableSize, rowsToLoad, totalRows, fields.length, this._materialisesRows);
|
|
3368
|
+
for (const field of allFields) {
|
|
2148
3369
|
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
2149
3370
|
}
|
|
2150
|
-
this._symbolTable = fields.map((field) => {
|
|
3371
|
+
this._symbolTable = fields.map((field, position) => {
|
|
3372
|
+
this._throwIfAborted();
|
|
2151
3373
|
const symbolsOffset = parseInt(field["Offset"], 10);
|
|
2152
3374
|
const symbolsLength = parseInt(field["Length"], 10);
|
|
2153
|
-
const
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
pointer,
|
|
2165
|
-
symbolBuffer.length,
|
|
2166
|
-
fieldName,
|
|
2167
|
-
this._path,
|
|
2168
|
-
shouldKeepSymbol
|
|
2169
|
-
);
|
|
2170
|
-
symbols.push(symbol);
|
|
2171
|
-
pointer += bytesRead - 1;
|
|
2172
|
-
symbolIndex++;
|
|
2173
|
-
}
|
|
3375
|
+
const symbols = parseFieldSymbols(
|
|
3376
|
+
symbolBuffer,
|
|
3377
|
+
symbolsOffset,
|
|
3378
|
+
symbolsOffset + symbolsLength,
|
|
3379
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
3380
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
3381
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
3382
|
+
field["FieldName"],
|
|
3383
|
+
this._path
|
|
3384
|
+
);
|
|
3385
|
+
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
2174
3386
|
return symbols;
|
|
2175
3387
|
});
|
|
2176
3388
|
}
|
|
@@ -2189,13 +3401,18 @@ var init_QvdFileReader = __esm({
|
|
|
2189
3401
|
* same for every row, so they are hoisted out of the loop and the inner loop does arithmetic
|
|
2190
3402
|
* into a typed array and nothing else. Rows are assembled later, once, in `load()`.
|
|
2191
3403
|
*
|
|
2192
|
-
*
|
|
3404
|
+
* The window is what makes chunked iteration cheap: `decodeIndexColumn` walks records by
|
|
3405
|
+
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3406
|
+
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3407
|
+
*
|
|
3408
|
+
* @param {QvdRowWindow} window The rows to decode.
|
|
2193
3409
|
*/
|
|
2194
|
-
async _parseIndexTable(
|
|
2195
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(
|
|
3410
|
+
async _parseIndexTable(window) {
|
|
3411
|
+
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
2196
3412
|
this._rowsDecoded = rowsToLoad;
|
|
2197
|
-
this._indexColumns = fields.map(
|
|
2198
|
-
(
|
|
3413
|
+
this._indexColumns = fields.map((field, position) => {
|
|
3414
|
+
this._throwIfAborted();
|
|
3415
|
+
const column = decodeIndexColumn(
|
|
2199
3416
|
indexBuffer,
|
|
2200
3417
|
recordSize,
|
|
2201
3418
|
rowsToLoad,
|
|
@@ -2203,8 +3420,10 @@ var init_QvdFileReader = __esm({
|
|
|
2203
3420
|
parseInt(field["BitWidth"], 10),
|
|
2204
3421
|
parseInt(field["Bias"], 10),
|
|
2205
3422
|
new Int32Array(rowsToLoad)
|
|
2206
|
-
)
|
|
2207
|
-
|
|
3423
|
+
);
|
|
3424
|
+
this._emitProgress("index-table", position + 1, fields.length);
|
|
3425
|
+
return column;
|
|
3426
|
+
});
|
|
2208
3427
|
}
|
|
2209
3428
|
/**
|
|
2210
3429
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -2221,8 +3440,11 @@ var init_QvdFileReader = __esm({
|
|
|
2221
3440
|
* @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
|
|
2222
3441
|
*/
|
|
2223
3442
|
async loadMetadata() {
|
|
2224
|
-
await this._readData(null, true);
|
|
3443
|
+
await this._readData({ offset: 0, limit: null }, true);
|
|
3444
|
+
this._emitProgress("header", 0, 1);
|
|
2225
3445
|
await this._parseHeader();
|
|
3446
|
+
this._emitProgress("header", 1, 1);
|
|
3447
|
+
this._throwIfAborted();
|
|
2226
3448
|
assert2(this._header, "The QVD file header has not been parsed.");
|
|
2227
3449
|
const header = this._header["QvdTableHeader"];
|
|
2228
3450
|
let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
|
|
@@ -2257,137 +3479,337 @@ var init_QvdFileReader = __esm({
|
|
|
2257
3479
|
/**
|
|
2258
3480
|
* Loads the QVD file into memory and parses it.
|
|
2259
3481
|
*
|
|
2260
|
-
* @param {number|null
|
|
2261
|
-
*
|
|
2262
|
-
*
|
|
3482
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3483
|
+
* The rows to load. A number or null means what it always meant - the first N rows, or all of
|
|
3484
|
+
* them - and `{offset, limit}` is the same thing said more precisely, so `5` and
|
|
3485
|
+
* `{offset: 0, limit: 5}` are one read. `maxRows` is accepted as a second name for `limit`.
|
|
3486
|
+
* @throws {QvdValidationError} If the window is not a non-negative integer, null, or a valid
|
|
3487
|
+
* `{offset, limit}` object.
|
|
2263
3488
|
* @return {Promise<QvdDataFrame>} The loaded QVD file.
|
|
2264
3489
|
*/
|
|
2265
|
-
async load(
|
|
2266
|
-
const
|
|
3490
|
+
async load(window = null) {
|
|
3491
|
+
const rows = normaliseWindow(window, this._path);
|
|
3492
|
+
const prepared = await this._prepare(rows);
|
|
3493
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3494
|
+
const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
|
|
3495
|
+
return new QvdDataFrame(
|
|
3496
|
+
data,
|
|
3497
|
+
prepared.columns,
|
|
3498
|
+
prepared.metadata,
|
|
3499
|
+
{
|
|
3500
|
+
...prepared.loadStats,
|
|
3501
|
+
rowsLoaded: data.length
|
|
3502
|
+
},
|
|
3503
|
+
prepared.storedSymbols
|
|
3504
|
+
);
|
|
3505
|
+
}
|
|
3506
|
+
/**
|
|
3507
|
+
* Reads the file as columns, without ever materialising rows.
|
|
3508
|
+
*
|
|
3509
|
+
* Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
|
|
3510
|
+
* What it keeps instead is what the decoder already produced: one `Int32Array` of stored
|
|
3511
|
+
* indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
|
|
3512
|
+
* fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
|
|
3513
|
+
* bytes per row rather than a boxed value per cell, and the symbols are a few thousand
|
|
3514
|
+
* entries shared across every row that uses them.
|
|
3515
|
+
*
|
|
3516
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3517
|
+
* The rows to decode, in the same spellings `load()` accepts.
|
|
3518
|
+
* @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
|
|
3519
|
+
*/
|
|
3520
|
+
async loadColumnar(window = null) {
|
|
3521
|
+
const rows = normaliseWindow(window, this._path);
|
|
3522
|
+
const prepared = await this._prepare(rows, null, true);
|
|
3523
|
+
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3524
|
+
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
2267
3525
|
assert2(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
}
|
|
2277
|
-
|
|
2278
|
-
}
|
|
2279
|
-
loadStats.rowsLoaded = data.length;
|
|
2280
|
-
return new QvdDataFrame(data, columns, metadata, loadStats);
|
|
3526
|
+
return new QvdColumnTable2({
|
|
3527
|
+
columns: prepared.columns,
|
|
3528
|
+
codesByField: this._indexColumns,
|
|
3529
|
+
symbolsByField: prepared.resolvedByField,
|
|
3530
|
+
halvesByField: prepared.halvesByField,
|
|
3531
|
+
rowCount: this._rowsDecoded,
|
|
3532
|
+
metadata: prepared.metadata,
|
|
3533
|
+
storedSymbols: prepared.storedSymbols,
|
|
3534
|
+
loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
|
|
3535
|
+
});
|
|
2281
3536
|
}
|
|
2282
3537
|
/**
|
|
2283
|
-
*
|
|
3538
|
+
* Yields the window as data frames of at most `chunkSize` rows.
|
|
2284
3539
|
*
|
|
2285
|
-
*
|
|
2286
|
-
*
|
|
2287
|
-
*
|
|
2288
|
-
*
|
|
2289
|
-
*
|
|
3540
|
+
* The file is opened, read and parsed **once**; only the index decode and the row building
|
|
3541
|
+
* happen per chunk. That is the whole reason this exists as a method rather than as a loop of
|
|
3542
|
+
* `load({offset, limit})` calls at the call site: the symbol table has to be parsed in full
|
|
3543
|
+
* whatever the chunk size - a stored index in the last chunk can address the first symbol -
|
|
3544
|
+
* and re-parsing it per chunk is what makes the obvious implementation cost more than a plain
|
|
3545
|
+
* load rather than less. PyQvd's chunked read does re-read it, and the comment on #140 records
|
|
3546
|
+
* that as a limitation rather than a design.
|
|
2290
3547
|
*
|
|
2291
|
-
*
|
|
2292
|
-
*
|
|
2293
|
-
*
|
|
2294
|
-
*
|
|
3548
|
+
* What it bounds is row materialisation, which is what actually dominates a large read's heap.
|
|
3549
|
+
* Two chunks of rows are alive at a time, not one - `for await` keeps the yielded frame
|
|
3550
|
+
* reachable while this generator builds the next - which is why `liveRows` below is
|
|
3551
|
+
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
3552
|
+
*
|
|
3553
|
+
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
3554
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
3555
|
+
*
|
|
3556
|
+
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
3557
|
+
* The rows to cover, in the same spellings `load()` accepts.
|
|
3558
|
+
* @param {number} chunkSize Rows per frame. Must be a positive integer.
|
|
3559
|
+
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in order.
|
|
2295
3560
|
*/
|
|
2296
|
-
async
|
|
2297
|
-
if (
|
|
2298
|
-
throw new QvdValidationError("
|
|
2299
|
-
provided:
|
|
2300
|
-
type: typeof
|
|
3561
|
+
async *iterateRows(window, chunkSize) {
|
|
3562
|
+
if (typeof chunkSize !== "number" || !Number.isInteger(chunkSize) || chunkSize <= 0) {
|
|
3563
|
+
throw new QvdValidationError("chunkSize must be a positive integer", {
|
|
3564
|
+
provided: chunkSize,
|
|
3565
|
+
type: typeof chunkSize,
|
|
2301
3566
|
file: this._path
|
|
2302
3567
|
});
|
|
2303
3568
|
}
|
|
2304
|
-
|
|
3569
|
+
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
3570
|
+
const rows = normaliseWindow(window, this._path);
|
|
3571
|
+
const prepared = await this._prepare(rows, liveRows);
|
|
3572
|
+
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
3573
|
+
this._throwIfAborted();
|
|
3574
|
+
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
3575
|
+
const offset = prepared.offset + done;
|
|
3576
|
+
await this._parseIndexTable({ offset, limit: count });
|
|
3577
|
+
const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
|
|
3578
|
+
yield new QvdDataFrame(
|
|
3579
|
+
data,
|
|
3580
|
+
prepared.columns,
|
|
3581
|
+
prepared.metadata,
|
|
3582
|
+
{
|
|
3583
|
+
...prepared.loadStats,
|
|
3584
|
+
offset,
|
|
3585
|
+
rowsLoaded: data.length
|
|
3586
|
+
},
|
|
3587
|
+
prepared.storedSymbols
|
|
3588
|
+
);
|
|
3589
|
+
}
|
|
3590
|
+
}
|
|
3591
|
+
/**
|
|
3592
|
+
* Reads the file and resolves its symbols, stopping short of decoding any rows.
|
|
3593
|
+
*
|
|
3594
|
+
* Everything `load()`, `loadColumnar()` and `iterateRows()` have in common, which is everything
|
|
3595
|
+
* that depends on the file rather than on the window. Two read paths for one binary format is
|
|
3596
|
+
* the drift risk #113 is the standing example of - a stored index resolved one way here and
|
|
3597
|
+
* another way there returns plausible wrong values and throws nothing - so there is one path,
|
|
3598
|
+
* and the entry points differ only in what they do with what it returns and how many rows they
|
|
3599
|
+
* ask for at a time.
|
|
3600
|
+
*
|
|
3601
|
+
* @param {QvdRowWindow} window The rows the read covers.
|
|
3602
|
+
* @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
|
|
3603
|
+
* is fewer than the window covers, and how many of them one row of the caller's chunk size
|
|
3604
|
+
* accounts for. Only `iterateRows` passes it; every other read holds what it covers.
|
|
3605
|
+
* @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
|
|
3606
|
+
* cells do not show them, which only a columnar read has a use for.
|
|
3607
|
+
* @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
|
|
3608
|
+
* resolvedByField: Array<Array<any>>,
|
|
3609
|
+
* halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
|
|
3610
|
+
* storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
|
|
3611
|
+
* rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
|
|
3612
|
+
* against it.
|
|
3613
|
+
* @private
|
|
3614
|
+
*/
|
|
3615
|
+
async _prepare(window, liveRows = null, wantHalves = false) {
|
|
3616
|
+
this._throwIfAborted();
|
|
3617
|
+
await this._readData(window, false, liveRows);
|
|
3618
|
+
this._emitProgress("header", 0, 1);
|
|
2305
3619
|
await this._parseHeader();
|
|
3620
|
+
this._emitProgress("header", 1, 1);
|
|
3621
|
+
this._throwIfAborted();
|
|
3622
|
+
assert2(this._header, "The QVD file header has not been parsed.");
|
|
3623
|
+
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
3624
|
+
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3625
|
+
const resolved = resolveWindow(window, totalRows);
|
|
3626
|
+
const rowsAvailable = resolved.limit;
|
|
2306
3627
|
let symbolsToKeep = null;
|
|
2307
3628
|
let symbolsKept = null;
|
|
2308
|
-
if (
|
|
2309
|
-
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3629
|
+
if (window.limit !== null || window.offset > 0) {
|
|
2310
3630
|
if (symbolTableLength > this._symbolFilteringThreshold) {
|
|
2311
|
-
symbolsToKeep = await this._analyzeIndexTableSymbolUsage(
|
|
2312
|
-
symbolsKept =
|
|
3631
|
+
symbolsToKeep = await this._analyzeIndexTableSymbolUsage({ offset: resolved.offset, limit: rowsAvailable });
|
|
3632
|
+
symbolsKept = symbolsToKeep.reduce((sum, set) => sum + set.size, 0);
|
|
2313
3633
|
}
|
|
2314
3634
|
}
|
|
2315
|
-
await this._parseSymbolTable(symbolsToKeep,
|
|
2316
|
-
await this._parseIndexTable(maxRows);
|
|
2317
|
-
assert2(this._header, "The QVD file header has not been parsed.");
|
|
3635
|
+
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
2318
3636
|
assert2(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
3637
|
+
this._throwIfAborted();
|
|
3638
|
+
assert2(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3639
|
+
const resolvedByField = [];
|
|
3640
|
+
const halvesByField = [];
|
|
3641
|
+
const entries = [];
|
|
3642
|
+
this._symbolTable.forEach((symbols, position) => {
|
|
3643
|
+
const { values, entry, halves } = resolveFieldSymbols(
|
|
3644
|
+
symbols,
|
|
3645
|
+
// @ts-ignore - asserted above
|
|
3646
|
+
this._selectedFields[position]["FieldName"],
|
|
3647
|
+
this._duals,
|
|
3648
|
+
this._coerceNumericStrings,
|
|
3649
|
+
wantHalves
|
|
3650
|
+
);
|
|
3651
|
+
resolvedByField.push(values);
|
|
3652
|
+
halvesByField.push(halves);
|
|
3653
|
+
if (entry !== null) {
|
|
3654
|
+
entries.push(entry);
|
|
2325
3655
|
}
|
|
2326
|
-
return resolved;
|
|
2327
3656
|
});
|
|
2328
|
-
|
|
2329
|
-
if (!Array.isArray(fields)) {
|
|
2330
|
-
fields = [fields];
|
|
2331
|
-
}
|
|
2332
|
-
const columns = fields.map((field) => field["FieldName"]);
|
|
3657
|
+
const columns = this._selectedFields.map((field) => field["FieldName"]);
|
|
2333
3658
|
const metadata = this._header["QvdTableHeader"];
|
|
3659
|
+
const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
|
|
3660
|
+
if (storedSymbols !== null) {
|
|
3661
|
+
attachStoredSymbols(metadata, storedSymbols);
|
|
3662
|
+
}
|
|
2334
3663
|
const loadStats = {
|
|
2335
|
-
symbolTableBytes:
|
|
2336
|
-
totalRows
|
|
2337
|
-
rowsLoaded:
|
|
3664
|
+
symbolTableBytes: symbolTableLength,
|
|
3665
|
+
totalRows,
|
|
3666
|
+
rowsLoaded: 0,
|
|
3667
|
+
offset: resolved.offset,
|
|
2338
3668
|
symbolFiltering: symbolsToKeep !== null,
|
|
2339
3669
|
symbolsKept
|
|
2340
3670
|
};
|
|
2341
|
-
return {
|
|
3671
|
+
return {
|
|
3672
|
+
columns,
|
|
3673
|
+
metadata,
|
|
3674
|
+
loadStats,
|
|
3675
|
+
resolvedByField,
|
|
3676
|
+
halvesByField,
|
|
3677
|
+
storedSymbols,
|
|
3678
|
+
rowsAvailable,
|
|
3679
|
+
offset: resolved.offset
|
|
3680
|
+
};
|
|
2342
3681
|
}
|
|
2343
3682
|
/**
|
|
2344
|
-
*
|
|
3683
|
+
* Builds rows from the columns currently decoded.
|
|
2345
3684
|
*
|
|
2346
|
-
*
|
|
2347
|
-
*
|
|
2348
|
-
*
|
|
2349
|
-
*
|
|
2350
|
-
*
|
|
2351
|
-
* entries shared across every row that uses them.
|
|
3685
|
+
* `data` stays eager: of the four ways this library is used - a full read, a preview already
|
|
3686
|
+
* bounded by a limit, writing an array out, and reading metadata - not one is helped by
|
|
3687
|
+
* materialising a row only when it is touched, and a lazy accessor would cost a proxy, a cache
|
|
3688
|
+
* and mutation semantics to serve none of them. A caller who wants columns without paying for
|
|
3689
|
+
* rows uses `QvdColumnTable`, which stops before this loop.
|
|
2352
3690
|
*
|
|
2353
|
-
* @param {
|
|
2354
|
-
* @
|
|
3691
|
+
* @param {Array<Array<any>>} resolvedByField One resolved value per distinct symbol, per field.
|
|
3692
|
+
* @param {number} progressBase Rows already delivered before this call, so that progress over a
|
|
3693
|
+
* chunked iteration counts the whole window rather than restarting at every chunk.
|
|
3694
|
+
* @param {number} progressTotal Rows the whole window covers.
|
|
3695
|
+
* @return {Array<Array<any>>} The rows.
|
|
3696
|
+
* @private
|
|
2355
3697
|
*/
|
|
2356
|
-
|
|
2357
|
-
const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
|
|
2358
|
-
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
3698
|
+
_buildRows(resolvedByField, progressBase, progressTotal) {
|
|
2359
3699
|
assert2(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
3700
|
+
const indexColumns = this._indexColumns;
|
|
3701
|
+
const fieldCount = indexColumns.length;
|
|
3702
|
+
const rowCount = this._rowsDecoded;
|
|
3703
|
+
const data = new Array(rowCount);
|
|
3704
|
+
const reportInterval = Math.max(1, Math.floor(progressTotal / 100));
|
|
3705
|
+
for (let row = 0; row < rowCount; row++) {
|
|
3706
|
+
const values = new Array(fieldCount);
|
|
3707
|
+
for (let field = 0; field < fieldCount; field++) {
|
|
3708
|
+
const symbolIndex = indexColumns[field][row];
|
|
3709
|
+
values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
|
|
3710
|
+
}
|
|
3711
|
+
data[row] = values;
|
|
3712
|
+
if ((progressBase + row + 1) % reportInterval === 0 || row + 1 === rowCount) {
|
|
3713
|
+
this._throwIfAborted();
|
|
3714
|
+
this._emitProgress("rows", progressBase + row + 1, progressTotal);
|
|
3715
|
+
}
|
|
3716
|
+
}
|
|
3717
|
+
return data;
|
|
2368
3718
|
}
|
|
2369
3719
|
};
|
|
2370
3720
|
}
|
|
2371
3721
|
});
|
|
2372
3722
|
|
|
2373
3723
|
// src/QvdDataFrame.js
|
|
3724
|
+
function defaultFieldHeader(fieldName) {
|
|
3725
|
+
return {
|
|
3726
|
+
FieldName: fieldName,
|
|
3727
|
+
BitOffset: 0,
|
|
3728
|
+
BitWidth: 0,
|
|
3729
|
+
Bias: 0,
|
|
3730
|
+
NoOfSymbols: 0,
|
|
3731
|
+
Offset: 0,
|
|
3732
|
+
Length: 0,
|
|
3733
|
+
Comment: "",
|
|
3734
|
+
NumberFormat: {
|
|
3735
|
+
Type: "UNKNOWN",
|
|
3736
|
+
nDec: "0",
|
|
3737
|
+
UseThou: "0",
|
|
3738
|
+
Fmt: "",
|
|
3739
|
+
Dec: "",
|
|
3740
|
+
Thou: ""
|
|
3741
|
+
},
|
|
3742
|
+
Tags: {}
|
|
3743
|
+
};
|
|
3744
|
+
}
|
|
3745
|
+
function defaultHeader(columns) {
|
|
3746
|
+
return {
|
|
3747
|
+
QvBuildNo: 50667,
|
|
3748
|
+
CreatorDoc: "",
|
|
3749
|
+
CreateUtcTime: "",
|
|
3750
|
+
SourceCreateUtcTime: "",
|
|
3751
|
+
SourceFileUtcTime: "",
|
|
3752
|
+
SourceFileSize: -1,
|
|
3753
|
+
StaleUtcTime: "",
|
|
3754
|
+
TableName: "",
|
|
3755
|
+
Fields: {
|
|
3756
|
+
QvdFieldHeader: columns.map(defaultFieldHeader)
|
|
3757
|
+
},
|
|
3758
|
+
NoOfRecords: 0,
|
|
3759
|
+
RecordByteSize: 0,
|
|
3760
|
+
Offset: 0,
|
|
3761
|
+
Length: 0,
|
|
3762
|
+
Compression: "",
|
|
3763
|
+
Comment: "",
|
|
3764
|
+
EncryptionInfo: "",
|
|
3765
|
+
TableTags: "",
|
|
3766
|
+
ProfilingData: "",
|
|
3767
|
+
Lineage: {}
|
|
3768
|
+
};
|
|
3769
|
+
}
|
|
2374
3770
|
var QvdDataFrame;
|
|
2375
3771
|
var init_QvdDataFrame = __esm({
|
|
2376
3772
|
"src/QvdDataFrame.js"() {
|
|
2377
3773
|
init_QvdErrors();
|
|
3774
|
+
init_cellRules();
|
|
3775
|
+
init_readOptions();
|
|
3776
|
+
init_storedSymbols();
|
|
3777
|
+
__name(defaultFieldHeader, "defaultFieldHeader");
|
|
3778
|
+
__name(defaultHeader, "defaultHeader");
|
|
2378
3779
|
QvdDataFrame = class _QvdDataFrame {
|
|
3780
|
+
static {
|
|
3781
|
+
__name(this, "QvdDataFrame");
|
|
3782
|
+
}
|
|
2379
3783
|
/**
|
|
2380
3784
|
* Represents the data frame stored inside a QVD file.
|
|
3785
|
+
*
|
|
3786
|
+
* The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
|
|
3787
|
+
* its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
|
|
3788
|
+
* Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
|
|
3789
|
+
* holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
|
|
3790
|
+
* header's record did for a frame built from some of a read's columns.
|
|
3791
|
+
*
|
|
2381
3792
|
* @param {Array<Array<any>>} data The data of the data frame.
|
|
2382
3793
|
* @param {Array<string>} columns The columns of the data frame.
|
|
2383
3794
|
* @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
|
|
2384
3795
|
* @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
|
|
3796
|
+
* @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
|
|
3797
|
+
* shows only one half of its symbol (optional) - see `storedSymbols`.
|
|
3798
|
+
* @throws {QvdValidationError} If the record is malformed.
|
|
2385
3799
|
*/
|
|
2386
|
-
constructor(data, columns, metadata = null, loadStats = null) {
|
|
3800
|
+
constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
|
|
2387
3801
|
this._data = data;
|
|
2388
3802
|
this._columns = columns;
|
|
2389
3803
|
this._metadata = metadata;
|
|
3804
|
+
this._ownsMetadata = false;
|
|
2390
3805
|
this._loadStats = loadStats;
|
|
3806
|
+
this._storedSymbols = narrowStoredSymbols(
|
|
3807
|
+
normaliseStoredSymbols(
|
|
3808
|
+
// @ts-ignore - a symbol-keyed property the reader defines on the header object
|
|
3809
|
+
storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
|
|
3810
|
+
),
|
|
3811
|
+
columns
|
|
3812
|
+
);
|
|
2391
3813
|
}
|
|
2392
3814
|
/**
|
|
2393
3815
|
* Returns the data of the data frame.
|
|
@@ -2414,15 +3836,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2414
3836
|
get metadata() {
|
|
2415
3837
|
return this._metadata;
|
|
2416
3838
|
}
|
|
3839
|
+
/**
|
|
3840
|
+
* What the frame's cells were read from, where a cell shows only one half of its symbol.
|
|
3841
|
+
*
|
|
3842
|
+
* A dual read as its number has a text the cell does not show; one read as its text has a number;
|
|
3843
|
+
* a string read as a number has the text it was spelled with. The record keeps those halves, per
|
|
3844
|
+
* field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
|
|
3845
|
+
* and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
|
|
3846
|
+
* `select`, `toDict` and `fromDict`.
|
|
3847
|
+
*
|
|
3848
|
+
* Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
|
|
3849
|
+
* the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
|
|
3850
|
+
* pure number.
|
|
3851
|
+
*
|
|
3852
|
+
* @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
|
|
3853
|
+
* showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
|
|
3854
|
+
* in the record it was given or found - one from `select`, or one built from some of a read's
|
|
3855
|
+
* columns and its header - has an empty record rather than null, because a frame given null takes
|
|
3856
|
+
* the record its header carries, which describes every field of the read.
|
|
3857
|
+
*/
|
|
3858
|
+
get storedSymbols() {
|
|
3859
|
+
return this._storedSymbols;
|
|
3860
|
+
}
|
|
2417
3861
|
/**
|
|
2418
3862
|
* Returns statistics about the read that produced this data frame.
|
|
2419
3863
|
*
|
|
2420
|
-
*
|
|
2421
|
-
*
|
|
3864
|
+
* Carried by every frame that came from a file - `fromQvd()`, and each chunk `iterate()` yields,
|
|
3865
|
+
* which is how a chunk reports its `offset`. `fromDict()`, `head()`, `tail()`, `rows()` and
|
|
3866
|
+
* `select()` describe no particular read and report null rather than a stale figure.
|
|
2422
3867
|
*
|
|
2423
3868
|
* The main use is confirming that a lazy load actually filtered the symbol table:
|
|
2424
3869
|
* `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
|
|
2425
|
-
* survived it
|
|
3870
|
+
* survived it. Note that a bounded read does not filter on its own - the two-pass path engages
|
|
3871
|
+
* only above `symbolFilteringThreshold`, so on a file below it this reports false and every
|
|
3872
|
+
* symbol was parsed however few rows were asked for.
|
|
2426
3873
|
*
|
|
2427
3874
|
* @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
|
|
2428
3875
|
*/
|
|
@@ -2531,54 +3978,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2531
3978
|
* @property {string} [profilingData] - Profiling data
|
|
2532
3979
|
* @property {Object|string} [lineage] - Lineage
|
|
2533
3980
|
*/
|
|
3981
|
+
/**
|
|
3982
|
+
* The header a metadata setter may change: this frame's own.
|
|
3983
|
+
*
|
|
3984
|
+
* A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
|
|
3985
|
+
* `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
|
|
3986
|
+
* change copies it, and a change made through one frame never reaches another. A frame with no header
|
|
3987
|
+
* gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
|
|
3988
|
+
* so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
|
|
3989
|
+
*
|
|
3990
|
+
* @return {any} The header.
|
|
3991
|
+
*/
|
|
3992
|
+
_ownMetadata() {
|
|
3993
|
+
if (!this._metadata) {
|
|
3994
|
+
this._metadata = defaultHeader(this._columns);
|
|
3995
|
+
} else if (!this._ownsMetadata) {
|
|
3996
|
+
const record = this._metadata[STORED_SYMBOLS];
|
|
3997
|
+
this._metadata = structuredClone(this._metadata);
|
|
3998
|
+
if (record) {
|
|
3999
|
+
attachStoredSymbols(this._metadata, record);
|
|
4000
|
+
}
|
|
4001
|
+
}
|
|
4002
|
+
this._ownsMetadata = true;
|
|
4003
|
+
return this._metadata;
|
|
4004
|
+
}
|
|
2534
4005
|
/**
|
|
2535
4006
|
* Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
|
|
4007
|
+
*
|
|
4008
|
+
* The change applies to this frame only, never to a frame it was derived from or shares a header
|
|
4009
|
+
* with.
|
|
4010
|
+
*
|
|
2536
4011
|
* @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
|
|
2537
4012
|
*/
|
|
2538
4013
|
setFileMetadata(metadata) {
|
|
2539
|
-
|
|
2540
|
-
this._metadata = {
|
|
2541
|
-
QvBuildNo: 50667,
|
|
2542
|
-
CreatorDoc: "",
|
|
2543
|
-
CreateUtcTime: "",
|
|
2544
|
-
SourceCreateUtcTime: "",
|
|
2545
|
-
SourceFileUtcTime: "",
|
|
2546
|
-
SourceFileSize: -1,
|
|
2547
|
-
StaleUtcTime: "",
|
|
2548
|
-
TableName: "",
|
|
2549
|
-
Fields: {
|
|
2550
|
-
QvdFieldHeader: this._columns.map((column) => ({
|
|
2551
|
-
FieldName: column,
|
|
2552
|
-
BitOffset: 0,
|
|
2553
|
-
BitWidth: 0,
|
|
2554
|
-
Bias: 0,
|
|
2555
|
-
NoOfSymbols: 0,
|
|
2556
|
-
Offset: 0,
|
|
2557
|
-
Length: 0,
|
|
2558
|
-
Comment: "",
|
|
2559
|
-
NumberFormat: {
|
|
2560
|
-
Type: "UNKNOWN",
|
|
2561
|
-
nDec: "0",
|
|
2562
|
-
UseThou: "0",
|
|
2563
|
-
Fmt: "",
|
|
2564
|
-
Dec: "",
|
|
2565
|
-
Thou: ""
|
|
2566
|
-
},
|
|
2567
|
-
Tags: {}
|
|
2568
|
-
}))
|
|
2569
|
-
},
|
|
2570
|
-
NoOfRecords: 0,
|
|
2571
|
-
RecordByteSize: 0,
|
|
2572
|
-
Offset: 0,
|
|
2573
|
-
Length: 0,
|
|
2574
|
-
Compression: "",
|
|
2575
|
-
Comment: "",
|
|
2576
|
-
EncryptionInfo: "",
|
|
2577
|
-
TableTags: "",
|
|
2578
|
-
ProfilingData: "",
|
|
2579
|
-
Lineage: {}
|
|
2580
|
-
};
|
|
2581
|
-
}
|
|
4014
|
+
const header = this._ownMetadata();
|
|
2582
4015
|
const modifiableFields = [
|
|
2583
4016
|
"qvBuildNo",
|
|
2584
4017
|
"creatorDoc",
|
|
@@ -2613,7 +4046,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2613
4046
|
};
|
|
2614
4047
|
modifiableFields.forEach((field) => {
|
|
2615
4048
|
if (metadata[field] !== void 0) {
|
|
2616
|
-
|
|
4049
|
+
header[fieldMapping[field]] = metadata[field];
|
|
2617
4050
|
}
|
|
2618
4051
|
});
|
|
2619
4052
|
}
|
|
@@ -2626,30 +4059,44 @@ var init_QvdDataFrame = __esm({
|
|
|
2626
4059
|
/**
|
|
2627
4060
|
* Sets modifiable field-level metadata for a specific field.
|
|
2628
4061
|
* Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
|
|
4062
|
+
*
|
|
4063
|
+
* Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
|
|
4064
|
+
* header does not describe. The change applies to this frame only, never to a frame it was derived
|
|
4065
|
+
* from or shares a header with.
|
|
4066
|
+
*
|
|
2629
4067
|
* @param {string} fieldName The name of the field.
|
|
2630
4068
|
* @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
|
|
4069
|
+
* @throws {QvdValidationError} If the frame has no column of that name.
|
|
2631
4070
|
*/
|
|
2632
4071
|
setFieldMetadata(fieldName, metadata) {
|
|
2633
|
-
if (!this.
|
|
2634
|
-
|
|
4072
|
+
if (!this._columns.includes(fieldName)) {
|
|
4073
|
+
throw new QvdValidationError(`Column '${fieldName}' does not exist`, {
|
|
4074
|
+
column: fieldName,
|
|
4075
|
+
availableColumns: this._columns
|
|
4076
|
+
});
|
|
2635
4077
|
}
|
|
2636
|
-
|
|
4078
|
+
const header = this._ownMetadata();
|
|
4079
|
+
if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
|
|
4080
|
+
header.Fields = { QvdFieldHeader: [] };
|
|
4081
|
+
}
|
|
4082
|
+
let fields = header.Fields.QvdFieldHeader;
|
|
2637
4083
|
if (!Array.isArray(fields)) {
|
|
2638
4084
|
fields = [fields];
|
|
2639
|
-
|
|
4085
|
+
header.Fields.QvdFieldHeader = fields;
|
|
2640
4086
|
}
|
|
2641
|
-
|
|
2642
|
-
if (
|
|
2643
|
-
|
|
4087
|
+
let field = fields.find((f) => f.FieldName === fieldName);
|
|
4088
|
+
if (!field) {
|
|
4089
|
+
field = defaultFieldHeader(fieldName);
|
|
4090
|
+
fields.push(field);
|
|
2644
4091
|
}
|
|
2645
4092
|
if (metadata.comment !== void 0) {
|
|
2646
|
-
|
|
4093
|
+
field.Comment = metadata.comment;
|
|
2647
4094
|
}
|
|
2648
4095
|
if (metadata.numberFormat !== void 0) {
|
|
2649
|
-
|
|
4096
|
+
field.NumberFormat = metadata.numberFormat;
|
|
2650
4097
|
}
|
|
2651
4098
|
if (metadata.tags !== void 0) {
|
|
2652
|
-
|
|
4099
|
+
field.Tags = metadata.tags;
|
|
2653
4100
|
}
|
|
2654
4101
|
}
|
|
2655
4102
|
/**
|
|
@@ -2666,7 +4113,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2666
4113
|
type: typeof n
|
|
2667
4114
|
});
|
|
2668
4115
|
}
|
|
2669
|
-
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
|
|
4116
|
+
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
|
|
2670
4117
|
}
|
|
2671
4118
|
/**
|
|
2672
4119
|
* Returns the last n rows of the data frame.
|
|
@@ -2682,7 +4129,13 @@ var init_QvdDataFrame = __esm({
|
|
|
2682
4129
|
type: typeof n
|
|
2683
4130
|
});
|
|
2684
4131
|
}
|
|
2685
|
-
return new _QvdDataFrame(
|
|
4132
|
+
return new _QvdDataFrame(
|
|
4133
|
+
n === 0 ? [] : this._data.slice(-n),
|
|
4134
|
+
this._columns,
|
|
4135
|
+
this._metadata,
|
|
4136
|
+
null,
|
|
4137
|
+
this._storedSymbols
|
|
4138
|
+
);
|
|
2686
4139
|
}
|
|
2687
4140
|
/**
|
|
2688
4141
|
* Returns the selected rows of the data frame.
|
|
@@ -2710,7 +4163,9 @@ var init_QvdDataFrame = __esm({
|
|
|
2710
4163
|
return new _QvdDataFrame(
|
|
2711
4164
|
args.map((index) => this._data[index]),
|
|
2712
4165
|
this._columns,
|
|
2713
|
-
this._metadata
|
|
4166
|
+
this._metadata,
|
|
4167
|
+
null,
|
|
4168
|
+
this._storedSymbols
|
|
2714
4169
|
);
|
|
2715
4170
|
}
|
|
2716
4171
|
/**
|
|
@@ -2722,6 +4177,50 @@ var init_QvdDataFrame = __esm({
|
|
|
2722
4177
|
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
2723
4178
|
*/
|
|
2724
4179
|
at(row, column) {
|
|
4180
|
+
const index = this._cellIndex(row, column);
|
|
4181
|
+
return this._data[row][index];
|
|
4182
|
+
}
|
|
4183
|
+
/**
|
|
4184
|
+
* Returns the text of the value at the specified row and column.
|
|
4185
|
+
*
|
|
4186
|
+
* The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
|
|
4187
|
+
* `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
|
|
4188
|
+
* or the string it was spelled as - and is null for a number that was stored as a pure number.
|
|
4189
|
+
*
|
|
4190
|
+
* ```js
|
|
4191
|
+
* const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
|
|
4192
|
+
* df.at(0, 'date'); // -52593
|
|
4193
|
+
* df.textAt(0, 'date'); // '1756-01-01'
|
|
4194
|
+
* ```
|
|
4195
|
+
*
|
|
4196
|
+
* @param {number} row The index of the row.
|
|
4197
|
+
* @param {string} column The name of the column.
|
|
4198
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
4199
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4200
|
+
*/
|
|
4201
|
+
textAt(row, column) {
|
|
4202
|
+
const index = this._cellIndex(row, column);
|
|
4203
|
+
const value = this._data[row][index];
|
|
4204
|
+
if (typeof value === "string") {
|
|
4205
|
+
return value;
|
|
4206
|
+
}
|
|
4207
|
+
if (typeof value === "number") {
|
|
4208
|
+
const entry = storedSymbolsEntry(this._storedSymbols, column);
|
|
4209
|
+
return entry === null ? null : storedTextOf(entry, value);
|
|
4210
|
+
}
|
|
4211
|
+
const dual = asDual(value);
|
|
4212
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
4213
|
+
}
|
|
4214
|
+
/**
|
|
4215
|
+
* Checks a row and a column name, and returns the column's position.
|
|
4216
|
+
*
|
|
4217
|
+
* @param {number} row The index of the row.
|
|
4218
|
+
* @param {string} column The name of the column.
|
|
4219
|
+
* @return {number} The column's position.
|
|
4220
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4221
|
+
* @private
|
|
4222
|
+
*/
|
|
4223
|
+
_cellIndex(row, column) {
|
|
2725
4224
|
if (typeof row !== "number" || !Number.isInteger(row)) {
|
|
2726
4225
|
throw new QvdValidationError("Row index must be an integer", {
|
|
2727
4226
|
provided: row,
|
|
@@ -2741,7 +4240,7 @@ var init_QvdDataFrame = __esm({
|
|
|
2741
4240
|
availableColumns: this._columns
|
|
2742
4241
|
});
|
|
2743
4242
|
}
|
|
2744
|
-
return this.
|
|
4243
|
+
return this._columns.indexOf(column);
|
|
2745
4244
|
}
|
|
2746
4245
|
/**
|
|
2747
4246
|
* Selects the specified columns from the data frame.
|
|
@@ -2762,15 +4261,34 @@ var init_QvdDataFrame = __esm({
|
|
|
2762
4261
|
const indices = args.map((arg) => this._columns.indexOf(arg));
|
|
2763
4262
|
const data = this._data.map((row) => indices.map((index) => row[index]));
|
|
2764
4263
|
const columns = indices.map((index) => this._columns[index]);
|
|
2765
|
-
return new _QvdDataFrame(data, columns, this._metadata);
|
|
4264
|
+
return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
|
|
2766
4265
|
}
|
|
2767
4266
|
/**
|
|
2768
4267
|
* Returns the data frame as a dictionary.
|
|
2769
4268
|
*
|
|
2770
|
-
*
|
|
4269
|
+
* Everything a frame needs to write the same file again, as plain data: the header, and the
|
|
4270
|
+
* stored-symbol record that says what cells showing one half of a symbol were read from. So
|
|
4271
|
+
* `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
|
|
4272
|
+
* `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
|
|
4273
|
+
*
|
|
4274
|
+
* @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
|
|
2771
4275
|
*/
|
|
2772
4276
|
async toDict() {
|
|
2773
|
-
return
|
|
4277
|
+
return this.toJSON();
|
|
4278
|
+
}
|
|
4279
|
+
/**
|
|
4280
|
+
* The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
|
|
4281
|
+
* `fromDict(JSON.parse(...))` can revive.
|
|
4282
|
+
*
|
|
4283
|
+
* @return {QvdDataFrameDict} The data frame as a dictionary.
|
|
4284
|
+
*/
|
|
4285
|
+
toJSON() {
|
|
4286
|
+
return {
|
|
4287
|
+
columns: this._columns,
|
|
4288
|
+
data: this._data,
|
|
4289
|
+
metadata: this._metadata,
|
|
4290
|
+
storedSymbols: this._storedSymbols
|
|
4291
|
+
};
|
|
2774
4292
|
}
|
|
2775
4293
|
/**
|
|
2776
4294
|
* Persists the data frame to a QVD file.
|
|
@@ -2799,6 +4317,33 @@ var init_QvdDataFrame = __esm({
|
|
|
2799
4317
|
* @param {Object} [options] Optional loading options.
|
|
2800
4318
|
* @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
|
|
2801
4319
|
* integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
|
|
4320
|
+
* This is the older name for `limit`; the two are the same option and passing both throws.
|
|
4321
|
+
* @param {number|null} [options.limit] Rows to read, counting from `offset`. The same number as
|
|
4322
|
+
* `maxRows`, spelled so that it reads correctly beside an offset.
|
|
4323
|
+
* @param {number} [options.offset=0] File row to start at. An offset past the end of the file
|
|
4324
|
+
* returns no rows rather than throwing, so a paging loop terminates on its own.
|
|
4325
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
4326
|
+
* appear in the result. Unselected fields have their symbols skipped entirely rather than
|
|
4327
|
+
* parsed and discarded. An unknown or repeated name throws.
|
|
4328
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
4329
|
+
* text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
|
|
4330
|
+
* `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
|
|
4331
|
+
* `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
|
|
4332
|
+
* implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
|
|
4333
|
+
* `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
|
|
4334
|
+
* double, a string and NULL read the same in every mode: a number, a string and null. Anything else
|
|
4335
|
+
* throws.
|
|
4336
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4337
|
+
* reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
|
|
4338
|
+
* reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
|
|
4339
|
+
* `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
|
|
4340
|
+
* `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
|
|
4341
|
+
* original string or dual back, and a value that two stored values read as is refused there
|
|
4342
|
+
* rather than written as either. Anything but a boolean throws.
|
|
4343
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
4344
|
+
* read proceeds - the same shape `toQvd`'s callback receives.
|
|
4345
|
+
* @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
|
|
4346
|
+
* which is a `DOMException` named `AbortError` unless you aborted with a reason of your own.
|
|
2802
4347
|
* @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
|
|
2803
4348
|
* must be within this directory, with symlinks resolved first, so a link inside it that points
|
|
2804
4349
|
* outside it is rejected. Defaults to the current working directory. To permit an entire
|
|
@@ -2811,17 +4356,64 @@ var init_QvdDataFrame = __esm({
|
|
|
2811
4356
|
* **Zero disables the memory check entirely.**
|
|
2812
4357
|
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
|
|
2813
4358
|
* a lazy load switches to the two-pass filtering path. Defaults to 50MB.
|
|
2814
|
-
* @throws {QvdValidationError} If
|
|
4359
|
+
* @throws {QvdValidationError} If a window option is not a non-negative integer, if both
|
|
4360
|
+
* `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
|
|
4361
|
+
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
2815
4362
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
2816
4363
|
*/
|
|
2817
4364
|
static async fromQvd(path3, options = {}) {
|
|
2818
4365
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
4366
|
+
return await new QvdFileReader2(path3, readerOptionsFrom(options)).load(windowFrom(options));
|
|
4367
|
+
}
|
|
4368
|
+
/**
|
|
4369
|
+
* Reads a QVD file in chunks, as an async generator of data frames.
|
|
4370
|
+
*
|
|
4371
|
+
* The file is opened, read and parsed once; only the index decode and the row building happen
|
|
4372
|
+
* per chunk, so what this bounds is row materialisation - the part that actually dominates a
|
|
4373
|
+
* large read's heap. It is **not** constant-memory reading of an arbitrarily large file: the
|
|
4374
|
+
* symbol table is parsed in full whatever the chunk size, because a stored index in the last
|
|
4375
|
+
* chunk can address the first symbol. On a high-cardinality file that table is the bulk of the
|
|
4376
|
+
* cost, and `readMetadata` is the only read that avoids it.
|
|
4377
|
+
*
|
|
4378
|
+
* ```js
|
|
4379
|
+
* for await (const chunk of QvdDataFrame.iterate('big.qvd', {chunkSize: 50_000})) {
|
|
4380
|
+
* process(chunk.data);
|
|
4381
|
+
* }
|
|
4382
|
+
* ```
|
|
4383
|
+
*
|
|
4384
|
+
* A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
|
|
4385
|
+
* run its body.
|
|
4386
|
+
*
|
|
4387
|
+
* Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
|
|
4388
|
+
* them, so a chunk written on its own writes the symbols its cells were read from.
|
|
4389
|
+
*
|
|
4390
|
+
* @param {string} path The path to the QVD file.
|
|
4391
|
+
* @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
|
|
4392
|
+
* @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
|
|
4393
|
+
* @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
|
|
4394
|
+
* @param {number|null} [options.limit] Rows to cover, counting from `offset`.
|
|
4395
|
+
* @param {number} [options.offset=0] File row to start at.
|
|
4396
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
|
|
4397
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
|
|
4398
|
+
* its text, or a frozen `QvdDual` holding both. Anything else throws.
|
|
4399
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4400
|
+
* reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
4401
|
+
* `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
|
|
4402
|
+
* `storedSymbols`. Anything but a boolean throws.
|
|
4403
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
|
|
4404
|
+
* over the rows counts the whole window, not each chunk.
|
|
4405
|
+
* @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
|
|
4406
|
+
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
4407
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
|
|
4408
|
+
* charged for two chunks of rows rather than the window. Zero disables the check.
|
|
4409
|
+
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
|
|
4410
|
+
* windowed read switches to two-pass filtering.
|
|
4411
|
+
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
4412
|
+
*/
|
|
4413
|
+
static async *iterate(path3, options = {}) {
|
|
4414
|
+
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4415
|
+
const reader = new QvdFileReader2(path3, readerOptionsFrom(options));
|
|
4416
|
+
yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
|
|
2825
4417
|
}
|
|
2826
4418
|
/**
|
|
2827
4419
|
* Reads a QVD file's schema and header metadata, without reading its data.
|
|
@@ -2845,17 +4437,28 @@ var init_QvdDataFrame = __esm({
|
|
|
2845
4437
|
* @param {Object} [options] Optional reading options.
|
|
2846
4438
|
* @param {string} [options.allowedDir] Optional allowed directory path, applied exactly as it
|
|
2847
4439
|
* is for `fromQvd`.
|
|
4440
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`, as on
|
|
4441
|
+
* the reads that return data. Only the `read` and `header` stages occur here; there are no
|
|
4442
|
+
* symbols to parse and no rows to build.
|
|
4443
|
+
* @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
|
|
2848
4444
|
* @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
|
|
2849
4445
|
*/
|
|
2850
4446
|
static async readMetadata(path3, options = {}) {
|
|
2851
4447
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2852
|
-
return await new QvdFileReader2(path3,
|
|
4448
|
+
return await new QvdFileReader2(path3, metadataOptionsFrom(options)).loadMetadata();
|
|
2853
4449
|
}
|
|
2854
4450
|
/**
|
|
2855
4451
|
* Constructs a data frame from a dictionary.
|
|
2856
4452
|
*
|
|
2857
|
-
*
|
|
4453
|
+
* Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
|
|
4454
|
+
* rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
|
|
4455
|
+
* spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
|
|
4456
|
+
* here rather than written.
|
|
4457
|
+
*
|
|
4458
|
+
* @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
|
|
2858
4459
|
* @return {Promise<QvdDataFrame>} The constructed data frame.
|
|
4460
|
+
* @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
|
|
4461
|
+
* or `storedSymbols` is malformed or names a field that is not one of the columns.
|
|
2859
4462
|
*/
|
|
2860
4463
|
static async fromDict(data) {
|
|
2861
4464
|
if (!data.columns) {
|
|
@@ -2868,20 +4471,290 @@ var init_QvdDataFrame = __esm({
|
|
|
2868
4471
|
data
|
|
2869
4472
|
});
|
|
2870
4473
|
}
|
|
2871
|
-
|
|
4474
|
+
const { metadata = null, storedSymbols = null } = data;
|
|
4475
|
+
if (metadata !== null && !isPlainObject(metadata)) {
|
|
4476
|
+
throw new QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
|
|
4477
|
+
type: typeof metadata
|
|
4478
|
+
});
|
|
4479
|
+
}
|
|
4480
|
+
const record = normaliseStoredSymbols(storedSymbols);
|
|
4481
|
+
if (record !== null) {
|
|
4482
|
+
const unknown = record.find((entry) => !data.columns.includes(entry.field));
|
|
4483
|
+
if (unknown !== void 0) {
|
|
4484
|
+
throw new QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
|
|
4485
|
+
field: unknown.field,
|
|
4486
|
+
availableColumns: data.columns
|
|
4487
|
+
});
|
|
4488
|
+
}
|
|
4489
|
+
}
|
|
4490
|
+
return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
|
|
2872
4491
|
}
|
|
2873
4492
|
};
|
|
2874
4493
|
}
|
|
2875
4494
|
});
|
|
2876
4495
|
|
|
4496
|
+
// src/QvdSymbol.js
|
|
4497
|
+
init_QvdErrors();
|
|
4498
|
+
init_cellRules();
|
|
4499
|
+
init_symbolBytes();
|
|
4500
|
+
function checkInteger(value, context) {
|
|
4501
|
+
if (typeof value === "number" && isStoredAsInt(value)) {
|
|
4502
|
+
return;
|
|
4503
|
+
}
|
|
4504
|
+
throw new QvdValidationError(
|
|
4505
|
+
`The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
|
|
4506
|
+
{ ...context, half: "integer", type: typeof value }
|
|
4507
|
+
);
|
|
4508
|
+
}
|
|
4509
|
+
__name(checkInteger, "checkInteger");
|
|
4510
|
+
var QvdSymbol = class _QvdSymbol {
|
|
4511
|
+
static {
|
|
4512
|
+
__name(this, "QvdSymbol");
|
|
4513
|
+
}
|
|
4514
|
+
/**
|
|
4515
|
+
* Constructs a new QVD symbol.
|
|
4516
|
+
*
|
|
4517
|
+
* @param {number|null} intValue The integer value.
|
|
4518
|
+
* @param {number|null} doubleValue The double value.
|
|
4519
|
+
* @param {string|null} stringValue The string value.
|
|
4520
|
+
*/
|
|
4521
|
+
constructor(intValue, doubleValue, stringValue) {
|
|
4522
|
+
this._intValue = intValue;
|
|
4523
|
+
this._doubleValue = doubleValue;
|
|
4524
|
+
this._stringValue = stringValue;
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Returns the integer value of this symbol.
|
|
4528
|
+
*
|
|
4529
|
+
* @return {number|null} The integer value.
|
|
4530
|
+
*/
|
|
4531
|
+
get intValue() {
|
|
4532
|
+
return this._intValue;
|
|
4533
|
+
}
|
|
4534
|
+
/**
|
|
4535
|
+
* Returns the double value of this symbol.
|
|
4536
|
+
*
|
|
4537
|
+
* @return {number|null} The double value.
|
|
4538
|
+
*/
|
|
4539
|
+
get doubleValue() {
|
|
4540
|
+
return this._doubleValue;
|
|
4541
|
+
}
|
|
4542
|
+
/**
|
|
4543
|
+
* Returns the string value of this symbol.
|
|
4544
|
+
*
|
|
4545
|
+
* @return {string|null} The string value.
|
|
4546
|
+
*/
|
|
4547
|
+
get stringValue() {
|
|
4548
|
+
return this._stringValue;
|
|
4549
|
+
}
|
|
4550
|
+
/**
|
|
4551
|
+
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
4552
|
+
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
4553
|
+
*
|
|
4554
|
+
* @return {number|string|null} The primary value.
|
|
4555
|
+
*/
|
|
4556
|
+
toPrimaryValue() {
|
|
4557
|
+
if (null != this._stringValue) {
|
|
4558
|
+
return this._stringValue;
|
|
4559
|
+
} else if (null != this._intValue) {
|
|
4560
|
+
return this._intValue;
|
|
4561
|
+
} else if (null != this._doubleValue) {
|
|
4562
|
+
return this._doubleValue;
|
|
4563
|
+
} else {
|
|
4564
|
+
return null;
|
|
4565
|
+
}
|
|
4566
|
+
}
|
|
4567
|
+
/**
|
|
4568
|
+
* Converts the symbol to its byte representation.
|
|
4569
|
+
*
|
|
4570
|
+
* The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
|
|
4571
|
+
* double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
|
|
4572
|
+
* checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
|
|
4573
|
+
* outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
|
|
4574
|
+
* integer and a double silently lost the double, a text with a NUL produced a symbol that ends
|
|
4575
|
+
* early, and a text with an unpaired surrogate was written with U+FFFD in its place.
|
|
4576
|
+
*
|
|
4577
|
+
* A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
|
|
4578
|
+
* `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
|
|
4579
|
+
* `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
|
|
4580
|
+
*
|
|
4581
|
+
* @return {Buffer} The byte representation of the symbol.
|
|
4582
|
+
* @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
|
|
4583
|
+
* holds a half no symbol can store. The message names the half.
|
|
4584
|
+
*/
|
|
4585
|
+
toByteRepresentation() {
|
|
4586
|
+
const intValue = this._intValue ?? null;
|
|
4587
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4588
|
+
const stringValue = this._stringValue ?? null;
|
|
4589
|
+
if (intValue !== null && doubleValue !== null) {
|
|
4590
|
+
throw new QvdValidationError("A symbol holds an integer or a double, not both", {
|
|
4591
|
+
intValue,
|
|
4592
|
+
doubleValue
|
|
4593
|
+
});
|
|
4594
|
+
}
|
|
4595
|
+
if (intValue === null && doubleValue === null && stringValue === null) {
|
|
4596
|
+
throw new QvdValidationError("The symbol does not contain any value.", {
|
|
4597
|
+
intValue,
|
|
4598
|
+
doubleValue,
|
|
4599
|
+
stringValue
|
|
4600
|
+
});
|
|
4601
|
+
}
|
|
4602
|
+
if (intValue !== null) {
|
|
4603
|
+
checkInteger(intValue, {});
|
|
4604
|
+
}
|
|
4605
|
+
if (doubleValue !== null) {
|
|
4606
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4607
|
+
}
|
|
4608
|
+
if (stringValue !== null) {
|
|
4609
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4610
|
+
}
|
|
4611
|
+
const number = intValue ?? doubleValue;
|
|
4612
|
+
const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
|
|
4613
|
+
const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
|
|
4614
|
+
writeSymbol(buffer, 0, kind, number, stringValue);
|
|
4615
|
+
return buffer;
|
|
4616
|
+
}
|
|
4617
|
+
/**
|
|
4618
|
+
* Checks if this symbol is equal to another symbol.
|
|
4619
|
+
*
|
|
4620
|
+
* By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
|
|
4621
|
+
* `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
|
|
4622
|
+
* answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
|
|
4623
|
+
* or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
|
|
4624
|
+
* storage kind, so either kind matches.
|
|
4625
|
+
*
|
|
4626
|
+
* @param {*} value The object to compare with.
|
|
4627
|
+
* @return {boolean} True if the objects are equal, false otherwise.
|
|
4628
|
+
*/
|
|
4629
|
+
equals(value) {
|
|
4630
|
+
if (value === null || typeof value !== "object") {
|
|
4631
|
+
return false;
|
|
4632
|
+
}
|
|
4633
|
+
const intValue = this._intValue ?? null;
|
|
4634
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4635
|
+
const stringValue = this._stringValue ?? null;
|
|
4636
|
+
const dual = asDual(value);
|
|
4637
|
+
if (dual !== null) {
|
|
4638
|
+
return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
|
|
4639
|
+
}
|
|
4640
|
+
if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
|
|
4641
|
+
return false;
|
|
4642
|
+
}
|
|
4643
|
+
return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
|
|
4644
|
+
}
|
|
4645
|
+
/**
|
|
4646
|
+
* Constructs a pure integer value symbol.
|
|
4647
|
+
*
|
|
4648
|
+
* @param {number} intValue The integer value.
|
|
4649
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4650
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
|
|
4651
|
+
*/
|
|
4652
|
+
static fromIntValue(intValue) {
|
|
4653
|
+
checkInteger(intValue, {});
|
|
4654
|
+
return new _QvdSymbol(intValue, null, null);
|
|
4655
|
+
}
|
|
4656
|
+
/**
|
|
4657
|
+
* Constructs a pure double value symbol.
|
|
4658
|
+
*
|
|
4659
|
+
* @param {number} doubleValue The double value.
|
|
4660
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4661
|
+
* @throws {QvdValidationError} If the double is not a finite number.
|
|
4662
|
+
*/
|
|
4663
|
+
static fromDoubleValue(doubleValue) {
|
|
4664
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4665
|
+
return new _QvdSymbol(null, doubleValue, null);
|
|
4666
|
+
}
|
|
4667
|
+
/**
|
|
4668
|
+
* Constructs a pure string value symbol.
|
|
4669
|
+
*
|
|
4670
|
+
* @param {string} stringValue The string value.
|
|
4671
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4672
|
+
* @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
|
|
4673
|
+
*/
|
|
4674
|
+
static fromStringValue(stringValue) {
|
|
4675
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4676
|
+
return new _QvdSymbol(null, null, stringValue);
|
|
4677
|
+
}
|
|
4678
|
+
/**
|
|
4679
|
+
* Constructs a dual value symbol from an integer and a string value.
|
|
4680
|
+
*
|
|
4681
|
+
* @param {number} intValue The integer value.
|
|
4682
|
+
* @param {string} stringValue The string value.
|
|
4683
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4684
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
|
|
4685
|
+
* is not a string, or holds a NUL or an unpaired surrogate.
|
|
4686
|
+
*/
|
|
4687
|
+
static fromDualIntValue(intValue, stringValue) {
|
|
4688
|
+
checkInteger(intValue, {});
|
|
4689
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4690
|
+
return new _QvdSymbol(intValue, null, stringValue);
|
|
4691
|
+
}
|
|
4692
|
+
/**
|
|
4693
|
+
* Constructs a dual value symbol from a double and a string value.
|
|
4694
|
+
*
|
|
4695
|
+
* @param {number} doubleValue The double value.
|
|
4696
|
+
* @param {string} stringValue The string value.
|
|
4697
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4698
|
+
* @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
|
|
4699
|
+
* or holds a NUL or an unpaired surrogate.
|
|
4700
|
+
*/
|
|
4701
|
+
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
4702
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4703
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4704
|
+
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
4705
|
+
}
|
|
4706
|
+
};
|
|
4707
|
+
|
|
4708
|
+
// src/index.js
|
|
4709
|
+
init_QvdDual();
|
|
4710
|
+
|
|
4711
|
+
// src/util/qlikDate.js
|
|
4712
|
+
init_QvdErrors();
|
|
4713
|
+
init_cellRules();
|
|
4714
|
+
var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
|
|
4715
|
+
var MS_PER_DAY = 864e5;
|
|
4716
|
+
var MAX_DATE_MS = 864e13;
|
|
4717
|
+
var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4718
|
+
var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4719
|
+
function qlikSerialToDate(serial) {
|
|
4720
|
+
const dual = asDual(serial);
|
|
4721
|
+
const number = dual === null ? serial : dual.number;
|
|
4722
|
+
if (typeof number !== "number" || !Number.isFinite(number)) {
|
|
4723
|
+
throw new QvdValidationError("A Qlik date serial must be a finite number", {
|
|
4724
|
+
provided: describeType(number),
|
|
4725
|
+
type: typeof serial
|
|
4726
|
+
});
|
|
4727
|
+
}
|
|
4728
|
+
const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
|
|
4729
|
+
if (!(Math.abs(ms) <= MAX_DATE_MS)) {
|
|
4730
|
+
throw new QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
|
|
4731
|
+
serial: number,
|
|
4732
|
+
minSerial: MIN_SERIAL,
|
|
4733
|
+
maxSerial: MAX_SERIAL
|
|
4734
|
+
});
|
|
4735
|
+
}
|
|
4736
|
+
return new Date(ms);
|
|
4737
|
+
}
|
|
4738
|
+
__name(qlikSerialToDate, "qlikSerialToDate");
|
|
4739
|
+
function dateToQlikSerial(date) {
|
|
4740
|
+
const ms = types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
|
|
4741
|
+
if (Number.isNaN(ms)) {
|
|
4742
|
+
throw new QvdValidationError("dateToQlikSerial needs a valid Date", {
|
|
4743
|
+
provided: describeType(date),
|
|
4744
|
+
type: typeof date
|
|
4745
|
+
});
|
|
4746
|
+
}
|
|
4747
|
+
return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4748
|
+
}
|
|
4749
|
+
__name(dateToQlikSerial, "dateToQlikSerial");
|
|
4750
|
+
|
|
2877
4751
|
// src/index.js
|
|
2878
|
-
init_QvdSymbol();
|
|
2879
4752
|
init_QvdDataFrame();
|
|
2880
4753
|
init_QvdColumnTable();
|
|
2881
4754
|
init_QvdFileReader();
|
|
2882
4755
|
init_QvdFileWriter();
|
|
2883
4756
|
init_QvdErrors();
|
|
2884
4757
|
|
|
2885
|
-
export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError };
|
|
4758
|
+
export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdDual, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError, dateToQlikSerial, qlikSerialToDate };
|
|
2886
4759
|
//# sourceMappingURL=index.js.map
|
|
2887
4760
|
//# sourceMappingURL=index.js.map
|