qvdjs 0.11.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -1318
- package/dist/index.cjs +1969 -572
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1967 -574
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
package/dist/index.cjs
CHANGED
|
@@ -7,6 +7,7 @@ var xml2 = require('xml2js');
|
|
|
7
7
|
var assert2 = require('assert');
|
|
8
8
|
var os = require('os');
|
|
9
9
|
var v8 = require('v8');
|
|
10
|
+
var util = require('util');
|
|
10
11
|
|
|
11
12
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
12
13
|
|
|
@@ -20,6 +21,7 @@ var v8__default = /*#__PURE__*/_interopDefault(v8);
|
|
|
20
21
|
|
|
21
22
|
var __defProp = Object.defineProperty;
|
|
22
23
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
24
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
23
25
|
var __esm = (fn, res) => function __init() {
|
|
24
26
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
25
27
|
};
|
|
@@ -29,10 +31,14 @@ var __export = (target, all) => {
|
|
|
29
31
|
};
|
|
30
32
|
|
|
31
33
|
// src/QvdErrors.js
|
|
32
|
-
exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
|
|
34
|
+
var ERROR_NAMES; exports.QvdError = void 0; exports.QvdParseError = void 0; exports.QvdValidationError = void 0; exports.QvdIOError = void 0; exports.QvdCorruptedError = void 0; exports.QvdSecurityError = void 0;
|
|
33
35
|
var init_QvdErrors = __esm({
|
|
34
36
|
"src/QvdErrors.js"() {
|
|
37
|
+
ERROR_NAMES = /* @__PURE__ */ new Map();
|
|
35
38
|
exports.QvdError = class extends Error {
|
|
39
|
+
static {
|
|
40
|
+
__name(this, "QvdError");
|
|
41
|
+
}
|
|
36
42
|
/**
|
|
37
43
|
* Constructs a new QVD error.
|
|
38
44
|
*
|
|
@@ -42,13 +48,16 @@ var init_QvdErrors = __esm({
|
|
|
42
48
|
*/
|
|
43
49
|
constructor(message, code, context = {}) {
|
|
44
50
|
super(message);
|
|
45
|
-
this.name =
|
|
51
|
+
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
46
52
|
this.code = code;
|
|
47
53
|
this.context = context;
|
|
48
54
|
Error.captureStackTrace(this, this.constructor);
|
|
49
55
|
}
|
|
50
56
|
};
|
|
51
57
|
exports.QvdParseError = class extends exports.QvdError {
|
|
58
|
+
static {
|
|
59
|
+
__name(this, "QvdParseError");
|
|
60
|
+
}
|
|
52
61
|
/**
|
|
53
62
|
* Constructs a new QVD parse error.
|
|
54
63
|
*
|
|
@@ -60,6 +69,9 @@ var init_QvdErrors = __esm({
|
|
|
60
69
|
}
|
|
61
70
|
};
|
|
62
71
|
exports.QvdValidationError = class extends exports.QvdError {
|
|
72
|
+
static {
|
|
73
|
+
__name(this, "QvdValidationError");
|
|
74
|
+
}
|
|
63
75
|
/**
|
|
64
76
|
* Constructs a new QVD validation error.
|
|
65
77
|
*
|
|
@@ -71,6 +83,9 @@ var init_QvdErrors = __esm({
|
|
|
71
83
|
}
|
|
72
84
|
};
|
|
73
85
|
exports.QvdIOError = class extends exports.QvdError {
|
|
86
|
+
static {
|
|
87
|
+
__name(this, "QvdIOError");
|
|
88
|
+
}
|
|
74
89
|
/**
|
|
75
90
|
* Constructs a new QVD IO error.
|
|
76
91
|
*
|
|
@@ -82,6 +97,9 @@ var init_QvdErrors = __esm({
|
|
|
82
97
|
}
|
|
83
98
|
};
|
|
84
99
|
exports.QvdCorruptedError = class extends exports.QvdError {
|
|
100
|
+
static {
|
|
101
|
+
__name(this, "QvdCorruptedError");
|
|
102
|
+
}
|
|
85
103
|
/**
|
|
86
104
|
* Constructs a new QVD corrupted error.
|
|
87
105
|
*
|
|
@@ -93,6 +111,9 @@ var init_QvdErrors = __esm({
|
|
|
93
111
|
}
|
|
94
112
|
};
|
|
95
113
|
exports.QvdSecurityError = class extends exports.QvdError {
|
|
114
|
+
static {
|
|
115
|
+
__name(this, "QvdSecurityError");
|
|
116
|
+
}
|
|
96
117
|
/**
|
|
97
118
|
* Constructs a new QVD security error.
|
|
98
119
|
*
|
|
@@ -103,163 +124,298 @@ var init_QvdErrors = __esm({
|
|
|
103
124
|
super(message, "QVD_SECURITY_ERROR", context);
|
|
104
125
|
}
|
|
105
126
|
};
|
|
127
|
+
ERROR_NAMES.set(exports.QvdError, "QvdError");
|
|
128
|
+
ERROR_NAMES.set(exports.QvdParseError, "QvdParseError");
|
|
129
|
+
ERROR_NAMES.set(exports.QvdValidationError, "QvdValidationError");
|
|
130
|
+
ERROR_NAMES.set(exports.QvdIOError, "QvdIOError");
|
|
131
|
+
ERROR_NAMES.set(exports.QvdCorruptedError, "QvdCorruptedError");
|
|
132
|
+
ERROR_NAMES.set(exports.QvdSecurityError, "QvdSecurityError");
|
|
106
133
|
}
|
|
107
134
|
});
|
|
108
135
|
|
|
109
|
-
// src/
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
136
|
+
// src/util/cellRules.js
|
|
137
|
+
function asDual(value) {
|
|
138
|
+
if (value === null || typeof value !== "object") {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
try {
|
|
142
|
+
if (value[DUAL_BRAND] === true) {
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
if (!isPlainObject(value)) {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
const keys = Object.keys(value);
|
|
149
|
+
return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
|
|
150
|
+
} catch {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function isPlainObject(value) {
|
|
155
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
const prototype = Object.getPrototypeOf(value);
|
|
159
|
+
return prototype === null || Object.getPrototypeOf(prototype) === null;
|
|
160
|
+
}
|
|
161
|
+
function isNumericText(text) {
|
|
162
|
+
return text.trim() !== "" && Number.isFinite(Number(text));
|
|
163
|
+
}
|
|
164
|
+
function isStoredAsInt(value) {
|
|
165
|
+
return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
|
|
166
|
+
}
|
|
167
|
+
function numberProblem(value) {
|
|
168
|
+
if (typeof value !== "number") {
|
|
169
|
+
return `is ${describeType(value)}, not a number`;
|
|
170
|
+
}
|
|
171
|
+
return Number.isFinite(value) ? null : `is ${String(value)}`;
|
|
172
|
+
}
|
|
173
|
+
function textProblem(value) {
|
|
174
|
+
if (typeof value !== "string") {
|
|
175
|
+
return { reason: "type" };
|
|
176
|
+
}
|
|
177
|
+
const position = value.indexOf(NUL);
|
|
178
|
+
if (position !== -1) {
|
|
179
|
+
return { reason: "nul", position };
|
|
180
|
+
}
|
|
181
|
+
return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
|
|
182
|
+
}
|
|
183
|
+
function unpairedSurrogateIndex(value) {
|
|
184
|
+
for (let index = 0; index < value.length; index++) {
|
|
185
|
+
const unit = value.charCodeAt(index);
|
|
186
|
+
if (unit < 55296 || unit > 57343) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const next = value.charCodeAt(index + 1);
|
|
190
|
+
if (unit <= 56319 && next >= 56320 && next <= 57343) {
|
|
191
|
+
index++;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
return index;
|
|
195
|
+
}
|
|
196
|
+
return -1;
|
|
197
|
+
}
|
|
198
|
+
function checkNumber(value, subject, context) {
|
|
199
|
+
if (numberProblem(value) === null) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (subject === null && typeof value === "number") {
|
|
203
|
+
throw new exports.QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
|
|
204
|
+
...context,
|
|
205
|
+
provided: String(value)
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
throw new exports.QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
|
|
209
|
+
...context,
|
|
210
|
+
type: typeof value
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
function checkText(value, subject, context) {
|
|
214
|
+
const problem = textProblem(value);
|
|
215
|
+
if (problem === null) {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (problem.reason === "type") {
|
|
219
|
+
throw new exports.QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
|
|
220
|
+
...context,
|
|
221
|
+
type: typeof value
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
if (problem.reason === "surrogate") {
|
|
225
|
+
throw new exports.QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
|
|
226
|
+
...context,
|
|
227
|
+
position: problem.position
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
throw new exports.QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
|
|
231
|
+
}
|
|
232
|
+
function constructorName(value) {
|
|
233
|
+
try {
|
|
234
|
+
const name = value.constructor?.name;
|
|
235
|
+
return typeof name === "string" && name !== "" ? name : null;
|
|
236
|
+
} catch {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
function describeType(value) {
|
|
241
|
+
if (value === null) {
|
|
242
|
+
return "null";
|
|
243
|
+
}
|
|
244
|
+
if (typeof value === "number") {
|
|
245
|
+
return Number.isFinite(value) ? "a number" : String(value);
|
|
246
|
+
}
|
|
247
|
+
if (typeof value === "undefined") {
|
|
248
|
+
return "undefined";
|
|
249
|
+
}
|
|
250
|
+
if (typeof value !== "object") {
|
|
251
|
+
return `a ${typeof value}`;
|
|
252
|
+
}
|
|
253
|
+
let name;
|
|
254
|
+
try {
|
|
255
|
+
if (Array.isArray(value)) {
|
|
256
|
+
return "an array";
|
|
257
|
+
}
|
|
258
|
+
name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
|
|
259
|
+
if (name === "Object") {
|
|
260
|
+
if (!isPlainObject(value)) {
|
|
261
|
+
return "an object whose prototype is not Object.prototype";
|
|
126
262
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
* @return {number|null} The integer value.
|
|
131
|
-
*/
|
|
132
|
-
get intValue() {
|
|
133
|
-
return this._intValue;
|
|
263
|
+
const keys = Object.keys(value);
|
|
264
|
+
if (keys.length === 0) {
|
|
265
|
+
return "a plain object";
|
|
134
266
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
267
|
+
return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
|
|
268
|
+
}
|
|
269
|
+
} catch {
|
|
270
|
+
return "an object";
|
|
271
|
+
}
|
|
272
|
+
return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
|
|
273
|
+
}
|
|
274
|
+
var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
|
|
275
|
+
var init_cellRules = __esm({
|
|
276
|
+
"src/util/cellRules.js"() {
|
|
277
|
+
init_QvdErrors();
|
|
278
|
+
INT32_MIN = -2147483648;
|
|
279
|
+
INT32_MAX = 2147483647;
|
|
280
|
+
NUL = String.fromCharCode(0);
|
|
281
|
+
DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
|
|
282
|
+
__name(asDual, "asDual");
|
|
283
|
+
__name(isPlainObject, "isPlainObject");
|
|
284
|
+
__name(isNumericText, "isNumericText");
|
|
285
|
+
__name(isStoredAsInt, "isStoredAsInt");
|
|
286
|
+
__name(numberProblem, "numberProblem");
|
|
287
|
+
__name(textProblem, "textProblem");
|
|
288
|
+
__name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
|
|
289
|
+
__name(checkNumber, "checkNumber");
|
|
290
|
+
__name(checkText, "checkText");
|
|
291
|
+
__name(constructorName, "constructorName");
|
|
292
|
+
__name(describeType, "describeType");
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
// src/util/symbolBytes.js
|
|
297
|
+
function kindOf(number, text) {
|
|
298
|
+
if (number === null) {
|
|
299
|
+
return 4;
|
|
300
|
+
}
|
|
301
|
+
if (isStoredAsInt(number)) {
|
|
302
|
+
return text === null ? 1 : 5;
|
|
303
|
+
}
|
|
304
|
+
return text === null ? 2 : 6;
|
|
305
|
+
}
|
|
306
|
+
function symbolByteLength(kind, number, text) {
|
|
307
|
+
const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
|
|
308
|
+
const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
|
|
309
|
+
return 1 + numberBytes + textBytes;
|
|
310
|
+
}
|
|
311
|
+
function writeSymbol(buffer, offset, kind, number, text) {
|
|
312
|
+
buffer[offset++] = kind;
|
|
313
|
+
if (kind === 1 || kind === 5) {
|
|
314
|
+
offset = buffer.writeInt32LE(number, offset);
|
|
315
|
+
} else if (kind === 2 || kind === 6) {
|
|
316
|
+
offset = buffer.writeDoubleLE(number, offset);
|
|
317
|
+
}
|
|
318
|
+
if (kind >= 4) {
|
|
319
|
+
offset += buffer.write(text, offset, "utf8");
|
|
320
|
+
buffer[offset++] = 0;
|
|
321
|
+
}
|
|
322
|
+
return offset;
|
|
323
|
+
}
|
|
324
|
+
var init_symbolBytes = __esm({
|
|
325
|
+
"src/util/symbolBytes.js"() {
|
|
326
|
+
init_cellRules();
|
|
327
|
+
__name(kindOf, "kindOf");
|
|
328
|
+
__name(symbolByteLength, "symbolByteLength");
|
|
329
|
+
__name(writeSymbol, "writeSymbol");
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
// src/QvdDual.js
|
|
334
|
+
function defineHalves(target, number, text) {
|
|
335
|
+
Object.defineProperties(target, {
|
|
336
|
+
number: { value: number, enumerable: true },
|
|
337
|
+
text: { value: text, enumerable: true }
|
|
338
|
+
});
|
|
339
|
+
Object.freeze(target);
|
|
340
|
+
}
|
|
341
|
+
function dualFromSymbol(number, text) {
|
|
342
|
+
const dual = Object.create(exports.QvdDual.prototype);
|
|
343
|
+
defineHalves(dual, number, text);
|
|
344
|
+
return dual;
|
|
345
|
+
}
|
|
346
|
+
exports.QvdDual = void 0;
|
|
347
|
+
var init_QvdDual = __esm({
|
|
348
|
+
"src/QvdDual.js"() {
|
|
349
|
+
init_cellRules();
|
|
350
|
+
exports.QvdDual = class {
|
|
351
|
+
static {
|
|
352
|
+
__name(this, "QvdDual");
|
|
142
353
|
}
|
|
143
354
|
/**
|
|
144
|
-
*
|
|
355
|
+
* Constructs a dual value.
|
|
145
356
|
*
|
|
146
|
-
*
|
|
147
|
-
|
|
148
|
-
get stringValue() {
|
|
149
|
-
return this._stringValue;
|
|
150
|
-
}
|
|
151
|
-
/**
|
|
152
|
-
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
153
|
-
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
357
|
+
* The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
|
|
358
|
+
* integer inside the int32 range is stored as a dual int, anything else as a dual double.
|
|
154
359
|
*
|
|
155
|
-
* @
|
|
360
|
+
* @param {number} number The numeric half. Must be a finite number.
|
|
361
|
+
* @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
|
|
362
|
+
* @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
|
|
156
363
|
*/
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
return this._intValue;
|
|
162
|
-
} else if (null != this._doubleValue) {
|
|
163
|
-
return this._doubleValue;
|
|
164
|
-
} else {
|
|
165
|
-
return null;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
/**
|
|
169
|
-
* Converts the symbol to its byte representation.
|
|
170
|
-
*
|
|
171
|
-
* @return {Buffer} The byte representation of the symbol.
|
|
172
|
-
*/
|
|
173
|
-
toByteRepresentation() {
|
|
174
|
-
if (this._intValue !== null && this._stringValue !== null) {
|
|
175
|
-
const intBuffer = Buffer.alloc(4);
|
|
176
|
-
intBuffer.writeInt32LE(this._intValue);
|
|
177
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
178
|
-
return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
|
|
179
|
-
} else if (this._doubleValue !== null && this._stringValue !== null) {
|
|
180
|
-
const floatBuffer = Buffer.alloc(8);
|
|
181
|
-
floatBuffer.writeDoubleLE(this._doubleValue);
|
|
182
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
183
|
-
return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
|
|
184
|
-
} else if (this._intValue !== null) {
|
|
185
|
-
const buffer = Buffer.alloc(4);
|
|
186
|
-
buffer.writeInt32LE(this._intValue);
|
|
187
|
-
return Buffer.concat([Buffer.from([1]), buffer]);
|
|
188
|
-
} else if (this._doubleValue !== null) {
|
|
189
|
-
const buffer = Buffer.alloc(8);
|
|
190
|
-
buffer.writeDoubleLE(this._doubleValue);
|
|
191
|
-
return Buffer.concat([Buffer.from([2]), buffer]);
|
|
192
|
-
} else if (this._stringValue !== null) {
|
|
193
|
-
const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
194
|
-
return Buffer.concat([Buffer.from([4]), buffer]);
|
|
195
|
-
} else {
|
|
196
|
-
throw new exports.QvdValidationError("The symbol does not contain any value.", {
|
|
197
|
-
intValue: this._intValue,
|
|
198
|
-
doubleValue: this._doubleValue,
|
|
199
|
-
stringValue: this._stringValue
|
|
200
|
-
});
|
|
201
|
-
}
|
|
364
|
+
constructor(number, text) {
|
|
365
|
+
checkNumber(number, "The number of a dual value", { half: "number" });
|
|
366
|
+
checkText(text, "The text of a dual value", { half: "text" });
|
|
367
|
+
defineHalves(this, number, text);
|
|
202
368
|
}
|
|
203
369
|
/**
|
|
204
|
-
*
|
|
370
|
+
* Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
|
|
205
371
|
*
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
*/
|
|
209
|
-
equals(value) {
|
|
210
|
-
if (!(value instanceof _QvdSymbol)) {
|
|
211
|
-
return false;
|
|
212
|
-
}
|
|
213
|
-
return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
|
|
214
|
-
}
|
|
215
|
-
/**
|
|
216
|
-
* Constructs a pure integer value symbol.
|
|
372
|
+
* The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
|
|
373
|
+
* conversion ran, not which half the caller meant, and the fix is the same for all three.
|
|
217
374
|
*
|
|
218
|
-
* @
|
|
219
|
-
* @
|
|
375
|
+
* @return {never}
|
|
376
|
+
* @throws {TypeError} Always.
|
|
220
377
|
*/
|
|
221
|
-
|
|
222
|
-
|
|
378
|
+
[Symbol.toPrimitive]() {
|
|
379
|
+
throw new TypeError(
|
|
380
|
+
`A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
|
|
381
|
+
);
|
|
223
382
|
}
|
|
224
383
|
/**
|
|
225
|
-
*
|
|
384
|
+
* Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
|
|
226
385
|
*
|
|
227
|
-
* @
|
|
228
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
386
|
+
* @return {{number: number, text: string}} The dual as a plain object.
|
|
229
387
|
*/
|
|
230
|
-
|
|
231
|
-
return
|
|
388
|
+
toJSON() {
|
|
389
|
+
return { number: this.number, text: this.text };
|
|
232
390
|
}
|
|
233
391
|
/**
|
|
234
|
-
*
|
|
392
|
+
* How Node's `util.inspect` and `console.log` show a dual.
|
|
235
393
|
*
|
|
236
|
-
* @
|
|
237
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
394
|
+
* @return {string} For example `QvdDual(4.5, "4.50")`.
|
|
238
395
|
*/
|
|
239
|
-
|
|
240
|
-
return
|
|
396
|
+
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
397
|
+
return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
|
|
241
398
|
}
|
|
242
|
-
/**
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
* @param {number} intValue The integer value.
|
|
246
|
-
* @param {string} stringValue The string value.
|
|
247
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
248
|
-
*/
|
|
249
|
-
static fromDualIntValue(intValue, stringValue) {
|
|
250
|
-
return new _QvdSymbol(intValue, null, stringValue);
|
|
399
|
+
/** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
|
|
400
|
+
get [Symbol.toStringTag]() {
|
|
401
|
+
return "QvdDual";
|
|
251
402
|
}
|
|
252
403
|
/**
|
|
253
|
-
*
|
|
404
|
+
* Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
|
|
405
|
+
* own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
|
|
406
|
+
*
|
|
407
|
+
* Recognition only. The halves are checked when the value is written.
|
|
254
408
|
*
|
|
255
|
-
* @param {
|
|
256
|
-
* @
|
|
257
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
409
|
+
* @param {any} value Any value.
|
|
410
|
+
* @return {boolean} True for a dual cell.
|
|
258
411
|
*/
|
|
259
|
-
static
|
|
260
|
-
return
|
|
412
|
+
static isDual(value) {
|
|
413
|
+
return asDual(value) !== null;
|
|
261
414
|
}
|
|
262
415
|
};
|
|
416
|
+
Object.defineProperty(exports.QvdDual.prototype, DUAL_BRAND, { value: true });
|
|
417
|
+
__name(defineHalves, "defineHalves");
|
|
418
|
+
__name(dualFromSymbol, "dualFromSymbol");
|
|
263
419
|
}
|
|
264
420
|
});
|
|
265
421
|
|
|
@@ -359,12 +515,41 @@ function selectFields(fields, requested, filePath) {
|
|
|
359
515
|
return fields[index];
|
|
360
516
|
});
|
|
361
517
|
}
|
|
518
|
+
function normaliseDuals(value, filePath) {
|
|
519
|
+
if (value === void 0 || value === null) {
|
|
520
|
+
return "number";
|
|
521
|
+
}
|
|
522
|
+
if (!DUAL_MODES.includes(value)) {
|
|
523
|
+
throw new exports.QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
524
|
+
option: "duals",
|
|
525
|
+
provided: value,
|
|
526
|
+
file: filePath
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
return value;
|
|
530
|
+
}
|
|
531
|
+
function normaliseCoerceNumericStrings(value, filePath) {
|
|
532
|
+
if (value === void 0 || value === null) {
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
if (typeof value !== "boolean") {
|
|
536
|
+
throw new exports.QvdValidationError("coerceNumericStrings must be true or false", {
|
|
537
|
+
option: "coerceNumericStrings",
|
|
538
|
+
provided: value,
|
|
539
|
+
type: typeof value,
|
|
540
|
+
file: filePath
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
return value;
|
|
544
|
+
}
|
|
362
545
|
function readerOptionsFrom(options) {
|
|
363
546
|
return {
|
|
364
547
|
allowedDir: options.allowedDir,
|
|
365
548
|
memorySafetyFactor: options.memorySafetyFactor,
|
|
366
549
|
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
367
550
|
fields: options.fields === void 0 ? null : options.fields,
|
|
551
|
+
duals: options.duals,
|
|
552
|
+
coerceNumericStrings: options.coerceNumericStrings,
|
|
368
553
|
onProgress: options.onProgress,
|
|
369
554
|
signal: options.signal
|
|
370
555
|
};
|
|
@@ -379,9 +564,151 @@ function metadataOptionsFrom(options) {
|
|
|
379
564
|
function windowFrom(options) {
|
|
380
565
|
return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
|
|
381
566
|
}
|
|
567
|
+
var DUAL_MODES;
|
|
382
568
|
var init_readOptions = __esm({
|
|
383
569
|
"src/util/readOptions.js"() {
|
|
384
570
|
init_QvdErrors();
|
|
571
|
+
__name(requireRowCount, "requireRowCount");
|
|
572
|
+
__name(normaliseWindow, "normaliseWindow");
|
|
573
|
+
__name(resolveWindow, "resolveWindow");
|
|
574
|
+
__name(selectFields, "selectFields");
|
|
575
|
+
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
576
|
+
__name(normaliseDuals, "normaliseDuals");
|
|
577
|
+
__name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
|
|
578
|
+
__name(readerOptionsFrom, "readerOptionsFrom");
|
|
579
|
+
__name(metadataOptionsFrom, "metadataOptionsFrom");
|
|
580
|
+
__name(windowFrom, "windowFrom");
|
|
581
|
+
}
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
// src/util/storedSymbols.js
|
|
585
|
+
function trustStoredSymbols(entries) {
|
|
586
|
+
const record = Object.freeze(entries);
|
|
587
|
+
trusted.add(record);
|
|
588
|
+
return record;
|
|
589
|
+
}
|
|
590
|
+
function attachStoredSymbols(metadata, record) {
|
|
591
|
+
if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
|
|
592
|
+
Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
function refuse(message, context) {
|
|
596
|
+
throw new exports.QvdValidationError(message, context);
|
|
597
|
+
}
|
|
598
|
+
function normaliseStoredSymbols(record) {
|
|
599
|
+
if (record === null || record === void 0) {
|
|
600
|
+
return null;
|
|
601
|
+
}
|
|
602
|
+
if (typeof record === "object" && trusted.has(record)) {
|
|
603
|
+
return record;
|
|
604
|
+
}
|
|
605
|
+
if (!Array.isArray(record)) {
|
|
606
|
+
refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
|
|
607
|
+
option: "storedSymbols",
|
|
608
|
+
type: typeof record
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
const fields = /* @__PURE__ */ new Set();
|
|
612
|
+
const entries = record.map((entry, index) => {
|
|
613
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
|
|
614
|
+
refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
|
|
615
|
+
}
|
|
616
|
+
const { field, values, numbers, texts } = entry;
|
|
617
|
+
if (fields.has(field)) {
|
|
618
|
+
refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
|
|
619
|
+
}
|
|
620
|
+
fields.add(field);
|
|
621
|
+
if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
|
|
622
|
+
refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
|
|
623
|
+
field,
|
|
624
|
+
entry: index
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
for (let symbol = 0; symbol < values.length; symbol++) {
|
|
628
|
+
const value = values[symbol];
|
|
629
|
+
const number = numbers[symbol];
|
|
630
|
+
const text = texts[symbol];
|
|
631
|
+
const context = { field, symbol };
|
|
632
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
633
|
+
refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
|
|
634
|
+
...context,
|
|
635
|
+
type: typeof value
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
if (number !== null) {
|
|
639
|
+
checkNumber(number, "A stored symbol's number", context);
|
|
640
|
+
}
|
|
641
|
+
if (text !== null) {
|
|
642
|
+
checkText(text, "A stored symbol's text", context);
|
|
643
|
+
}
|
|
644
|
+
const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
|
|
645
|
+
if (!consistent) {
|
|
646
|
+
refuse(
|
|
647
|
+
number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
|
|
648
|
+
{ ...context, value, number, text }
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
return Object.freeze({
|
|
653
|
+
field,
|
|
654
|
+
values: Object.freeze(values.slice()),
|
|
655
|
+
numbers: Object.freeze(numbers.slice()),
|
|
656
|
+
texts: Object.freeze(texts.slice())
|
|
657
|
+
});
|
|
658
|
+
});
|
|
659
|
+
return trustStoredSymbols(entries);
|
|
660
|
+
}
|
|
661
|
+
function narrowStoredSymbols(record, columns) {
|
|
662
|
+
if (record === null) {
|
|
663
|
+
return null;
|
|
664
|
+
}
|
|
665
|
+
const kept = record.filter((entry) => columns.includes(entry.field));
|
|
666
|
+
return kept.length === record.length ? record : trustStoredSymbols(kept);
|
|
667
|
+
}
|
|
668
|
+
function storedSymbolsEntry(record, field) {
|
|
669
|
+
if (record === null) {
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
return record.find((entry) => entry.field === field) ?? null;
|
|
673
|
+
}
|
|
674
|
+
function firstTextByValue(entry) {
|
|
675
|
+
const byValue = /* @__PURE__ */ new Map();
|
|
676
|
+
for (let index = entry.values.length - 1; index >= 0; index--) {
|
|
677
|
+
const text = entry.texts[index];
|
|
678
|
+
if (text !== null) {
|
|
679
|
+
byValue.set(entry.values[index], text);
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
return byValue;
|
|
683
|
+
}
|
|
684
|
+
function storedTextOf(entry, value) {
|
|
685
|
+
let byValue = firstTexts.get(entry);
|
|
686
|
+
if (byValue === void 0) {
|
|
687
|
+
byValue = firstTextByValue(entry);
|
|
688
|
+
firstTexts.set(entry, byValue);
|
|
689
|
+
}
|
|
690
|
+
return byValue.get(value) ?? null;
|
|
691
|
+
}
|
|
692
|
+
function sameValueZero(a, b) {
|
|
693
|
+
return a === b || a !== a && b !== b;
|
|
694
|
+
}
|
|
695
|
+
var STORED_SYMBOLS, trusted, firstTexts;
|
|
696
|
+
var init_storedSymbols = __esm({
|
|
697
|
+
"src/util/storedSymbols.js"() {
|
|
698
|
+
init_QvdErrors();
|
|
699
|
+
init_cellRules();
|
|
700
|
+
STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
|
|
701
|
+
trusted = /* @__PURE__ */ new WeakSet();
|
|
702
|
+
__name(trustStoredSymbols, "trustStoredSymbols");
|
|
703
|
+
__name(attachStoredSymbols, "attachStoredSymbols");
|
|
704
|
+
__name(refuse, "refuse");
|
|
705
|
+
__name(normaliseStoredSymbols, "normaliseStoredSymbols");
|
|
706
|
+
__name(narrowStoredSymbols, "narrowStoredSymbols");
|
|
707
|
+
__name(storedSymbolsEntry, "storedSymbolsEntry");
|
|
708
|
+
firstTexts = /* @__PURE__ */ new WeakMap();
|
|
709
|
+
__name(firstTextByValue, "firstTextByValue");
|
|
710
|
+
__name(storedTextOf, "storedTextOf");
|
|
711
|
+
__name(sameValueZero, "sameValueZero");
|
|
385
712
|
}
|
|
386
713
|
});
|
|
387
714
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
@@ -421,7 +748,7 @@ function resolveDeepestExisting(target) {
|
|
|
421
748
|
function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
422
749
|
let baseStat;
|
|
423
750
|
try {
|
|
424
|
-
baseStat = fs__default.default.statSync(fs__default.default.realpathSync(resolvedBaseDir));
|
|
751
|
+
baseStat = fs__default.default.statSync(fs__default.default.realpathSync(resolvedBaseDir), { bigint: true });
|
|
425
752
|
} catch {
|
|
426
753
|
return null;
|
|
427
754
|
}
|
|
@@ -432,7 +759,7 @@ function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
|
432
759
|
for (; ; ) {
|
|
433
760
|
let stat;
|
|
434
761
|
try {
|
|
435
|
-
stat = fs__default.default.statSync(current);
|
|
762
|
+
stat = fs__default.default.statSync(current, { bigint: true });
|
|
436
763
|
} catch {
|
|
437
764
|
return null;
|
|
438
765
|
}
|
|
@@ -486,6 +813,10 @@ function validatePath(filePath, allowedDir) {
|
|
|
486
813
|
var init_validatePath = __esm({
|
|
487
814
|
"src/util/validatePath.js"() {
|
|
488
815
|
init_QvdErrors();
|
|
816
|
+
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
817
|
+
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
818
|
+
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
819
|
+
__name(validatePath, "validatePath");
|
|
489
820
|
}
|
|
490
821
|
});
|
|
491
822
|
|
|
@@ -535,6 +866,9 @@ var init_bitUtils = __esm({
|
|
|
535
866
|
"src/util/bitUtils.js"() {
|
|
536
867
|
MAX_BIT_WIDTH = 31;
|
|
537
868
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
869
|
+
__name(fieldGeometry, "fieldGeometry");
|
|
870
|
+
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
871
|
+
__name(writeBitField, "writeBitField");
|
|
538
872
|
}
|
|
539
873
|
});
|
|
540
874
|
|
|
@@ -543,14 +877,310 @@ var QvdFileWriter_exports = {};
|
|
|
543
877
|
__export(QvdFileWriter_exports, {
|
|
544
878
|
QvdFileWriter: () => exports.QvdFileWriter
|
|
545
879
|
});
|
|
546
|
-
|
|
880
|
+
function notXmlIndex(value) {
|
|
881
|
+
for (let index = 0; index < value.length; index++) {
|
|
882
|
+
const unit = value.charCodeAt(index);
|
|
883
|
+
if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
|
|
884
|
+
return index;
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
return -1;
|
|
888
|
+
}
|
|
889
|
+
function checkHeaderText(value, subject, context) {
|
|
890
|
+
checkText(value, subject, context);
|
|
891
|
+
const position = notXmlIndex(value);
|
|
892
|
+
if (position !== -1) {
|
|
893
|
+
const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
|
|
894
|
+
throw new exports.QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
function checkHeaderTexts(value, property, owner, context) {
|
|
898
|
+
if (typeof value === "string") {
|
|
899
|
+
checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
|
|
900
|
+
} else if (Array.isArray(value)) {
|
|
901
|
+
value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
|
|
902
|
+
} else if (value !== null && typeof value === "object") {
|
|
903
|
+
for (const [key, item] of Object.entries(value)) {
|
|
904
|
+
checkHeaderTexts(item, `${property}.${key}`, owner, context);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
function validateColumnNames(columns, filePath) {
|
|
909
|
+
const seen = /* @__PURE__ */ new Set();
|
|
910
|
+
columns.forEach((name, index) => {
|
|
911
|
+
if (typeof name !== "string" || name.length === 0) {
|
|
912
|
+
throw new exports.QvdValidationError("Field names must be non-empty strings", {
|
|
913
|
+
column: index,
|
|
914
|
+
provided: name,
|
|
915
|
+
type: typeof name,
|
|
916
|
+
file: filePath,
|
|
917
|
+
stage: "buildSymbolTable"
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
|
|
921
|
+
if (seen.has(name)) {
|
|
922
|
+
throw new exports.QvdValidationError(`Field '${name}' appears twice`, {
|
|
923
|
+
column: name,
|
|
924
|
+
columnIndex: index,
|
|
925
|
+
file: filePath,
|
|
926
|
+
stage: "buildSymbolTable"
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
seen.add(name);
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
function refuseCell(value, column, row, filePath) {
|
|
933
|
+
let resemblesDual = false;
|
|
934
|
+
try {
|
|
935
|
+
resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
|
|
936
|
+
} catch {
|
|
937
|
+
}
|
|
938
|
+
throw new exports.QvdValidationError(
|
|
939
|
+
`A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
|
|
940
|
+
{
|
|
941
|
+
column,
|
|
942
|
+
row,
|
|
943
|
+
type: typeof value,
|
|
944
|
+
constructor: constructorName(value) ?? void 0,
|
|
945
|
+
file: filePath,
|
|
946
|
+
stage: "buildSymbolTable"
|
|
947
|
+
}
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
function valuesAreNumbers(entry) {
|
|
951
|
+
const { values, numbers } = entry;
|
|
952
|
+
for (let index = 0; index < values.length; index++) {
|
|
953
|
+
if (!sameValueZero(values[index], numbers[index])) {
|
|
954
|
+
return false;
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return true;
|
|
958
|
+
}
|
|
959
|
+
function newSlot(column, key, text) {
|
|
960
|
+
const slot = column.keys.length;
|
|
961
|
+
column.keys.push(key);
|
|
962
|
+
column.texts.push(text);
|
|
963
|
+
if (typeof key === "number") {
|
|
964
|
+
column.facts.hasNumber = true;
|
|
965
|
+
if (!Number.isInteger(key)) column.facts.hasFraction = true;
|
|
966
|
+
} else {
|
|
967
|
+
column.facts.hasNonNumber = true;
|
|
968
|
+
}
|
|
969
|
+
return slot;
|
|
970
|
+
}
|
|
971
|
+
function slotFor(column, key, text) {
|
|
972
|
+
let slot = column.byKey.get(key);
|
|
973
|
+
if (slot === void 0) {
|
|
974
|
+
slot = newSlot(column, key, text);
|
|
975
|
+
column.byKey.set(key, slot);
|
|
976
|
+
} else if (text !== null && column.texts[slot] === null && typeof key === "number") {
|
|
977
|
+
column.texts[slot] = text;
|
|
978
|
+
}
|
|
979
|
+
return slot;
|
|
980
|
+
}
|
|
981
|
+
function standsForSeveral(indices, numbers, texts) {
|
|
982
|
+
let number = null;
|
|
983
|
+
let string = null;
|
|
984
|
+
for (const index of indices) {
|
|
985
|
+
if (numbers[index] !== null) {
|
|
986
|
+
if (number === null) {
|
|
987
|
+
number = numbers[index];
|
|
988
|
+
} else if (!sameValueZero(number, numbers[index])) {
|
|
989
|
+
return true;
|
|
990
|
+
}
|
|
991
|
+
} else if (string === null) {
|
|
992
|
+
string = texts[index];
|
|
993
|
+
} else if (string !== texts[index]) {
|
|
994
|
+
return true;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
return number !== null && string !== null;
|
|
998
|
+
}
|
|
999
|
+
function ambiguousAsUncoercedText(entry) {
|
|
1000
|
+
const { values, numbers, texts } = entry;
|
|
1001
|
+
for (let index = 0; index < values.length; index++) {
|
|
1002
|
+
if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
|
|
1003
|
+
if (!isNumericText(texts[index])) {
|
|
1004
|
+
return false;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
const byShown = /* @__PURE__ */ new Map();
|
|
1009
|
+
for (let index = 0; index < values.length; index++) {
|
|
1010
|
+
const shown = texts[index] ?? numbers[index];
|
|
1011
|
+
const indices = byShown.get(shown);
|
|
1012
|
+
if (indices === void 0) {
|
|
1013
|
+
byShown.set(shown, [index]);
|
|
1014
|
+
} else {
|
|
1015
|
+
indices.push(index);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
for (const indices of byShown.values()) {
|
|
1019
|
+
if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
|
|
1020
|
+
return true;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
return false;
|
|
1024
|
+
}
|
|
1025
|
+
function unambiguousWithoutDuals(column) {
|
|
1026
|
+
const { numbers, texts } = column.entry;
|
|
1027
|
+
for (const found of column.byValue.values()) {
|
|
1028
|
+
if (typeof found !== "number") {
|
|
1029
|
+
const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
|
|
1030
|
+
if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
|
|
1031
|
+
return false;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
return true;
|
|
1036
|
+
}
|
|
1037
|
+
function ambiguityRemedy(column) {
|
|
1038
|
+
const { values, numbers, texts } = column.entry;
|
|
1039
|
+
let coerced = false;
|
|
1040
|
+
let other = false;
|
|
1041
|
+
for (const [value, found] of column.byValue) {
|
|
1042
|
+
if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
|
|
1043
|
+
if (typeof value === "number") {
|
|
1044
|
+
coerced = true;
|
|
1045
|
+
} else {
|
|
1046
|
+
other = true;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
if (!coerced) {
|
|
1051
|
+
return "Read the field with {duals: 'both'}, or write QvdDual cells.";
|
|
1052
|
+
}
|
|
1053
|
+
if (!other && !ambiguousAsUncoercedText(column.entry)) {
|
|
1054
|
+
return "Read the field without {coerceNumericStrings: true}.";
|
|
1055
|
+
}
|
|
1056
|
+
if (unambiguousWithoutDuals(column)) {
|
|
1057
|
+
return "Read the field with {duals: 'both'}.";
|
|
1058
|
+
}
|
|
1059
|
+
return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
|
|
1060
|
+
}
|
|
1061
|
+
function slotForCell(column, value, row, filePath) {
|
|
1062
|
+
if (column.byCell === column.byKey) {
|
|
1063
|
+
return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
|
|
1064
|
+
}
|
|
1065
|
+
const found = column.byValue.get(value);
|
|
1066
|
+
if (found === void 0) {
|
|
1067
|
+
return slotFor(column, value, null);
|
|
1068
|
+
}
|
|
1069
|
+
const { numbers, texts } = column.entry;
|
|
1070
|
+
const indices = typeof found === "number" ? [found] : found;
|
|
1071
|
+
if (standsForSeveral(indices, numbers, texts)) {
|
|
1072
|
+
const stored = [];
|
|
1073
|
+
for (const index of indices) {
|
|
1074
|
+
if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
|
|
1075
|
+
stored.push({ number: numbers[index], text: texts[index] });
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
throw new exports.QvdValidationError(
|
|
1079
|
+
`The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
|
|
1080
|
+
{ column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
|
|
1081
|
+
);
|
|
1082
|
+
}
|
|
1083
|
+
let number = null;
|
|
1084
|
+
let numberText = null;
|
|
1085
|
+
for (const index of indices) {
|
|
1086
|
+
if (numbers[index] === null) {
|
|
1087
|
+
return slotFor(column, texts[index], null);
|
|
1088
|
+
}
|
|
1089
|
+
number ??= numbers[index];
|
|
1090
|
+
numberText ??= texts[index];
|
|
1091
|
+
}
|
|
1092
|
+
return slotFor(column, number, numberText);
|
|
1093
|
+
}
|
|
1094
|
+
function slotForObject(column, value, row, filePath) {
|
|
1095
|
+
const dual = asDual(value);
|
|
1096
|
+
if (dual === null) {
|
|
1097
|
+
refuseCell(value, column.name, row, filePath);
|
|
1098
|
+
}
|
|
1099
|
+
const { number, text } = dual;
|
|
1100
|
+
if (numberProblem(number) !== null) {
|
|
1101
|
+
checkNumber(number, "The number of a dual value", {
|
|
1102
|
+
column: column.name,
|
|
1103
|
+
row,
|
|
1104
|
+
half: "number",
|
|
1105
|
+
file: filePath,
|
|
1106
|
+
stage: "buildSymbolTable"
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
if (textProblem(text) !== null) {
|
|
1110
|
+
checkText(text, "The text of a dual value", {
|
|
1111
|
+
column: column.name,
|
|
1112
|
+
row,
|
|
1113
|
+
half: "text",
|
|
1114
|
+
file: filePath,
|
|
1115
|
+
stage: "buildSymbolTable"
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
const slot = slotFor(column, number, text);
|
|
1119
|
+
if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
|
|
1120
|
+
column.byObject.set(value, slot);
|
|
1121
|
+
}
|
|
1122
|
+
return slot;
|
|
1123
|
+
}
|
|
1124
|
+
function pruneContradictedTags(tags, facts) {
|
|
1125
|
+
if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
|
|
1126
|
+
return tags || {};
|
|
1127
|
+
}
|
|
1128
|
+
const list = Array.isArray(tags.String) ? tags.String : [tags.String];
|
|
1129
|
+
const kept = list.filter((tag) => {
|
|
1130
|
+
if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
|
|
1131
|
+
if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
|
|
1132
|
+
if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
|
|
1133
|
+
return true;
|
|
1134
|
+
});
|
|
1135
|
+
if (kept.length === list.length) {
|
|
1136
|
+
return tags;
|
|
1137
|
+
}
|
|
1138
|
+
return kept.length === 0 ? {} : { ...tags, String: kept };
|
|
1139
|
+
}
|
|
1140
|
+
function resetContradictedNumberFormat(numberFormat, facts) {
|
|
1141
|
+
if (!numberFormat) {
|
|
1142
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1143
|
+
}
|
|
1144
|
+
if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
|
|
1145
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1146
|
+
}
|
|
1147
|
+
return numberFormat;
|
|
1148
|
+
}
|
|
1149
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT; exports.QvdFileWriter = void 0;
|
|
547
1150
|
var init_QvdFileWriter = __esm({
|
|
548
1151
|
"src/QvdFileWriter.js"() {
|
|
549
|
-
init_QvdSymbol();
|
|
550
1152
|
init_QvdErrors();
|
|
551
1153
|
init_validatePath();
|
|
552
1154
|
init_bitUtils();
|
|
553
|
-
|
|
1155
|
+
init_cellRules();
|
|
1156
|
+
init_symbolBytes();
|
|
1157
|
+
init_storedSymbols();
|
|
1158
|
+
__name(notXmlIndex, "notXmlIndex");
|
|
1159
|
+
__name(checkHeaderText, "checkHeaderText");
|
|
1160
|
+
__name(checkHeaderTexts, "checkHeaderTexts");
|
|
1161
|
+
__name(validateColumnNames, "validateColumnNames");
|
|
1162
|
+
__name(refuseCell, "refuseCell");
|
|
1163
|
+
OBJECT_MEMO_BASE = 64;
|
|
1164
|
+
__name(valuesAreNumbers, "valuesAreNumbers");
|
|
1165
|
+
__name(newSlot, "newSlot");
|
|
1166
|
+
__name(slotFor, "slotFor");
|
|
1167
|
+
__name(standsForSeveral, "standsForSeveral");
|
|
1168
|
+
__name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
|
|
1169
|
+
__name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
|
|
1170
|
+
__name(ambiguityRemedy, "ambiguityRemedy");
|
|
1171
|
+
__name(slotForCell, "slotForCell");
|
|
1172
|
+
__name(slotForObject, "slotForObject");
|
|
1173
|
+
NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
|
|
1174
|
+
TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
|
|
1175
|
+
WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
|
|
1176
|
+
NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
|
|
1177
|
+
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1178
|
+
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1179
|
+
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1180
|
+
exports.QvdFileWriter = class {
|
|
1181
|
+
static {
|
|
1182
|
+
__name(this, "QvdFileWriter");
|
|
1183
|
+
}
|
|
554
1184
|
/**
|
|
555
1185
|
* Constructs a new QVD file writer.
|
|
556
1186
|
*
|
|
@@ -571,7 +1201,8 @@ var init_QvdFileWriter = __esm({
|
|
|
571
1201
|
this._onProgress = onProgress;
|
|
572
1202
|
this._header = null;
|
|
573
1203
|
this._symbolBuffer = null;
|
|
574
|
-
this.
|
|
1204
|
+
this._symbolCounts = null;
|
|
1205
|
+
this._symbolFacts = null;
|
|
575
1206
|
this._symbolTableMetadata = null;
|
|
576
1207
|
this._indexBuffer = null;
|
|
577
1208
|
this._symbolIndexByValue = null;
|
|
@@ -682,19 +1313,12 @@ var init_QvdFileWriter = __esm({
|
|
|
682
1313
|
BitOffset: this._indexTableMetadata?.[index][0],
|
|
683
1314
|
BitWidth: this._indexTableMetadata?.[index][1],
|
|
684
1315
|
Bias: this._indexTableMetadata?.[index][2],
|
|
685
|
-
NoOfSymbols: this.
|
|
1316
|
+
NoOfSymbols: this._symbolCounts?.[index],
|
|
686
1317
|
Offset: this._symbolTableMetadata?.[index][0],
|
|
687
1318
|
Length: this._symbolTableMetadata?.[index][1],
|
|
688
1319
|
Comment: existingField?.Comment || "",
|
|
689
|
-
NumberFormat: existingField?.NumberFormat
|
|
690
|
-
|
|
691
|
-
nDec: "0",
|
|
692
|
-
UseThou: "0",
|
|
693
|
-
Fmt: "",
|
|
694
|
-
Dec: "",
|
|
695
|
-
Thou: ""
|
|
696
|
-
},
|
|
697
|
-
Tags: existingField?.Tags || {}
|
|
1320
|
+
NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
|
|
1321
|
+
Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
|
|
698
1322
|
};
|
|
699
1323
|
})
|
|
700
1324
|
},
|
|
@@ -704,6 +1328,19 @@ var init_QvdFileWriter = __esm({
|
|
|
704
1328
|
Length: this._indexBuffer?.length
|
|
705
1329
|
}
|
|
706
1330
|
};
|
|
1331
|
+
const { Fields, ...table } = xmlObject.QvdTableHeader;
|
|
1332
|
+
for (const [property, value] of Object.entries(table)) {
|
|
1333
|
+
checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
|
|
1334
|
+
}
|
|
1335
|
+
for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
|
|
1336
|
+
for (const [property, value] of Object.entries(field)) {
|
|
1337
|
+
checkHeaderTexts(value, property, `field '${FieldName}'`, {
|
|
1338
|
+
column: FieldName,
|
|
1339
|
+
file: this._path,
|
|
1340
|
+
stage: "buildHeader"
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
707
1344
|
const builder = new xml2__default.default.Builder({
|
|
708
1345
|
renderOpts: {
|
|
709
1346
|
pretty: true,
|
|
@@ -717,25 +1354,49 @@ var init_QvdFileWriter = __esm({
|
|
|
717
1354
|
/**
|
|
718
1355
|
* Builds the symbol table of the QVD file.
|
|
719
1356
|
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
*
|
|
1357
|
+
* One pass over the rows finds each column's distinct values in the order they first appear,
|
|
1358
|
+
* which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
|
|
1359
|
+
* column encodes them: every symbol is sized first, then written into one buffer of exactly that
|
|
1360
|
+
* size.
|
|
723
1361
|
*
|
|
724
|
-
*
|
|
725
|
-
* 1. Initialize a Set for each column to collect unique values
|
|
726
|
-
* 2. Single pass through all data rows, adding values to corresponding Sets
|
|
727
|
-
* 3. Convert Sets to arrays and create QvdSymbol instances
|
|
728
|
-
* 4. Serialize symbols to binary format and update metadata
|
|
1362
|
+
* What each value is stored as:
|
|
729
1363
|
*
|
|
730
|
-
*
|
|
731
|
-
*
|
|
732
|
-
*
|
|
733
|
-
* -
|
|
1364
|
+
* | Cell | Symbol |
|
|
1365
|
+
* | --- | --- |
|
|
1366
|
+
* | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
|
|
1367
|
+
* | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
|
|
1368
|
+
* | any other finite number | pure double, type 2 |
|
|
1369
|
+
* | a string | pure string, type 4 |
|
|
1370
|
+
* | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
|
|
1371
|
+
* | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
|
|
1372
|
+
*
|
|
1373
|
+
* A number is a pure number, with no text. It used to be written as a dual whose text was
|
|
1374
|
+
* `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
|
|
1375
|
+
* file Qlik wrote with pure numbers came back out of a read and a write with every one of them
|
|
1376
|
+
* turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
|
|
1377
|
+
* A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
|
|
1378
|
+
* UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
|
|
1379
|
+
* an unpaired surrogate has no UTF-8 encoding at all.
|
|
1380
|
+
*
|
|
1381
|
+
* A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
|
|
1382
|
+
* symbol with the first text in row order, and a plain number with the same number as a dual joins
|
|
1383
|
+
* the dual, in either order - so no text a caller supplied is lost to a plain number that came
|
|
1384
|
+
* first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
|
|
1385
|
+
*
|
|
1386
|
+
* A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
|
|
1387
|
+
* string read as a number. Its record says what each such cell stands for, so the frame writes back
|
|
1388
|
+
* the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
|
|
1389
|
+
* rows the cells were moved to. A cell the record maps to more than one stored value is refused.
|
|
1390
|
+
*
|
|
1391
|
+
* The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
|
|
1392
|
+
* for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
|
|
1393
|
+
* repository; `benchmarks/` measures what the writer costs now, which is the useful number.
|
|
734
1394
|
*
|
|
735
1395
|
* @private
|
|
736
1396
|
*/
|
|
737
1397
|
_buildSymbolTable() {
|
|
738
|
-
this.
|
|
1398
|
+
this._symbolCounts = [];
|
|
1399
|
+
this._symbolFacts = [];
|
|
739
1400
|
this._symbolTableMetadata = [];
|
|
740
1401
|
this._symbolIndexByValue = [];
|
|
741
1402
|
if (this._df.columns.length === 0) {
|
|
@@ -748,33 +1409,144 @@ var init_QvdFileWriter = __esm({
|
|
|
748
1409
|
const data = this._df.data;
|
|
749
1410
|
const numColumns = columns.length;
|
|
750
1411
|
const numRows = data.length;
|
|
1412
|
+
validateColumnNames(columns, this._path);
|
|
1413
|
+
const record = normaliseStoredSymbols(this._df.storedSymbols);
|
|
751
1414
|
this._emitProgress("symbol-table", 0, numColumns);
|
|
752
|
-
const
|
|
1415
|
+
const state = columns.map((name) => {
|
|
1416
|
+
const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
|
|
1417
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1418
|
+
let byValue = null;
|
|
1419
|
+
let textByNumber = null;
|
|
1420
|
+
if (entry !== null && valuesAreNumbers(entry)) {
|
|
1421
|
+
textByNumber = firstTextByValue(entry);
|
|
1422
|
+
} else if (entry !== null) {
|
|
1423
|
+
byValue = /* @__PURE__ */ new Map();
|
|
1424
|
+
for (let index = 0; index < entry.values.length; index++) {
|
|
1425
|
+
const found = byValue.get(entry.values[index]);
|
|
1426
|
+
if (found === void 0) {
|
|
1427
|
+
byValue.set(entry.values[index], index);
|
|
1428
|
+
} else if (typeof found === "number") {
|
|
1429
|
+
byValue.set(entry.values[index], [found, index]);
|
|
1430
|
+
} else {
|
|
1431
|
+
found.push(index);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
return {
|
|
1436
|
+
name,
|
|
1437
|
+
keys: [],
|
|
1438
|
+
texts: [],
|
|
1439
|
+
byKey,
|
|
1440
|
+
// A cell is its own key without a record entry, and with one whose values are their numbers, so
|
|
1441
|
+
// one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
|
|
1442
|
+
byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
|
|
1443
|
+
byObject: /* @__PURE__ */ new Map(),
|
|
1444
|
+
entry,
|
|
1445
|
+
textByNumber,
|
|
1446
|
+
byValue,
|
|
1447
|
+
facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
|
|
1448
|
+
};
|
|
1449
|
+
});
|
|
1450
|
+
const byCells = state.map((column) => column.byCell);
|
|
1451
|
+
const byObjects = state.map((column) => column.byObject);
|
|
753
1452
|
const containsNull = columns.map(() => false);
|
|
754
1453
|
for (let row = 0; row < numRows; row++) {
|
|
755
1454
|
const values = data[row];
|
|
1455
|
+
if (values !== null && values !== void 0 && !Array.isArray(values)) {
|
|
1456
|
+
throw new exports.QvdValidationError("Each row must be an array of values", {
|
|
1457
|
+
row,
|
|
1458
|
+
type: typeof values,
|
|
1459
|
+
file: this._path,
|
|
1460
|
+
stage: "buildSymbolTable"
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
if (values !== null && values !== void 0 && values.length > numColumns) {
|
|
1464
|
+
throw new exports.QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
|
|
1465
|
+
row,
|
|
1466
|
+
values: values.length,
|
|
1467
|
+
fields: numColumns,
|
|
1468
|
+
file: this._path,
|
|
1469
|
+
stage: "buildSymbolTable"
|
|
1470
|
+
});
|
|
1471
|
+
}
|
|
756
1472
|
for (let column = 0; column < numColumns; column++) {
|
|
757
1473
|
const value = values?.[column];
|
|
758
1474
|
if (value === null || value === void 0) {
|
|
759
1475
|
containsNull[column] = true;
|
|
760
1476
|
continue;
|
|
761
1477
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
1478
|
+
if (typeof value === "object") {
|
|
1479
|
+
if (!byObjects[column].has(value)) {
|
|
1480
|
+
slotForObject(state[column], value, row, this._path);
|
|
1481
|
+
}
|
|
1482
|
+
continue;
|
|
1483
|
+
}
|
|
1484
|
+
if (byCells[column].has(value)) {
|
|
1485
|
+
continue;
|
|
1486
|
+
}
|
|
1487
|
+
if (typeof value === "number") {
|
|
1488
|
+
if (numberProblem(value) !== null) {
|
|
1489
|
+
checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
|
|
1490
|
+
}
|
|
1491
|
+
} else if (typeof value === "string") {
|
|
1492
|
+
if (textProblem(value) !== null) {
|
|
1493
|
+
checkText(value, "A string value", {
|
|
1494
|
+
column: columns[column],
|
|
1495
|
+
row,
|
|
1496
|
+
file: this._path,
|
|
1497
|
+
stage: "buildSymbolTable"
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
} else {
|
|
1501
|
+
refuseCell(value, columns[column], row, this._path);
|
|
765
1502
|
}
|
|
1503
|
+
byCells[column].set(value, slotForCell(state[column], value, row, this._path));
|
|
766
1504
|
}
|
|
767
1505
|
}
|
|
768
1506
|
const columnBuffers = [];
|
|
769
1507
|
let symbolsOffset = 0;
|
|
770
1508
|
for (let column = 0; column < numColumns; column++) {
|
|
771
|
-
const
|
|
772
|
-
const
|
|
1509
|
+
const { keys, texts } = state[column];
|
|
1510
|
+
const kinds = new Uint8Array(keys.length);
|
|
1511
|
+
let byteLength = 0;
|
|
1512
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1513
|
+
const key = keys[slot];
|
|
1514
|
+
const text = texts[slot];
|
|
1515
|
+
if (typeof key === "number") {
|
|
1516
|
+
if (numberProblem(key) !== null) {
|
|
1517
|
+
checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
|
|
1518
|
+
}
|
|
1519
|
+
if (text !== null && textProblem(text) !== null) {
|
|
1520
|
+
checkText(text, "The text of a dual value", {
|
|
1521
|
+
column: columns[column],
|
|
1522
|
+
file: this._path,
|
|
1523
|
+
stage: "buildSymbolTable"
|
|
1524
|
+
});
|
|
1525
|
+
}
|
|
1526
|
+
kinds[slot] = kindOf(key, text);
|
|
1527
|
+
byteLength += symbolByteLength(kinds[slot], key, text);
|
|
1528
|
+
} else {
|
|
1529
|
+
kinds[slot] = kindOf(null, key);
|
|
1530
|
+
byteLength += symbolByteLength(kinds[slot], null, key);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
const columnBuffer = Buffer.allocUnsafe(byteLength);
|
|
1534
|
+
let offset = 0;
|
|
1535
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1536
|
+
const key = keys[slot];
|
|
1537
|
+
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1538
|
+
}
|
|
1539
|
+
assert2__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
773
1540
|
columnBuffers.push(columnBuffer);
|
|
774
|
-
this._symbolTableMetadata?.push([symbolsOffset,
|
|
775
|
-
this.
|
|
776
|
-
this.
|
|
777
|
-
|
|
1541
|
+
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1542
|
+
this._symbolCounts?.push(keys.length);
|
|
1543
|
+
this._symbolFacts?.push(state[column].facts);
|
|
1544
|
+
this._symbolIndexByValue?.push({
|
|
1545
|
+
byCell: state[column].byCell,
|
|
1546
|
+
byKey: state[column].byKey,
|
|
1547
|
+
byObject: state[column].byObject
|
|
1548
|
+
});
|
|
1549
|
+
symbolsOffset += byteLength;
|
|
778
1550
|
this._emitProgress("symbol-table", column + 1, numColumns);
|
|
779
1551
|
}
|
|
780
1552
|
this._symbolBuffer = Buffer.concat(columnBuffers);
|
|
@@ -803,7 +1575,7 @@ var init_QvdFileWriter = __esm({
|
|
|
803
1575
|
* @private
|
|
804
1576
|
*/
|
|
805
1577
|
_buildIndexTable() {
|
|
806
|
-
assert2__default.default(this.
|
|
1578
|
+
assert2__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
807
1579
|
assert2__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
808
1580
|
assert2__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
809
1581
|
this._indexTableMetadata = [];
|
|
@@ -817,20 +1589,24 @@ var init_QvdFileWriter = __esm({
|
|
|
817
1589
|
let totalBits = 0;
|
|
818
1590
|
for (let column = 0; column < numColumns; column++) {
|
|
819
1591
|
const fieldContainsNull = this._symbolTableMetadata[column][2];
|
|
820
|
-
const symbolCount = this.
|
|
1592
|
+
const symbolCount = this._symbolCounts[column];
|
|
821
1593
|
const nullShift = fieldContainsNull ? 2 : 0;
|
|
822
1594
|
const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
|
|
823
1595
|
const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
1596
|
+
const lookup = this._symbolIndexByValue[column];
|
|
1597
|
+
const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
|
|
1598
|
+
for (const map of maps) {
|
|
1599
|
+
for (const index of map.values()) {
|
|
1600
|
+
if (index + nullShift > maxStoredIndex) {
|
|
1601
|
+
throw new exports.QvdValidationError("The symbol table and the index table are out of sync", {
|
|
1602
|
+
field: columns[column],
|
|
1603
|
+
storedIndex: index + nullShift,
|
|
1604
|
+
maxStoredIndex,
|
|
1605
|
+
symbolCount,
|
|
1606
|
+
file: this._path,
|
|
1607
|
+
stage: "buildIndexTable"
|
|
1608
|
+
});
|
|
1609
|
+
}
|
|
834
1610
|
}
|
|
835
1611
|
}
|
|
836
1612
|
layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
|
|
@@ -841,6 +1617,9 @@ var init_QvdFileWriter = __esm({
|
|
|
841
1617
|
this._recordByteSize = recordByteSize;
|
|
842
1618
|
this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
|
|
843
1619
|
const progressInterval = Math.max(1, Math.floor(numRows / 100));
|
|
1620
|
+
const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
|
|
1621
|
+
const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
|
|
1622
|
+
const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
|
|
844
1623
|
for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
|
|
845
1624
|
const values = data[row];
|
|
846
1625
|
for (let column = 0; column < numColumns; column++) {
|
|
@@ -848,7 +1627,7 @@ var init_QvdFileWriter = __esm({
|
|
|
848
1627
|
if (value === null || value === void 0) {
|
|
849
1628
|
continue;
|
|
850
1629
|
}
|
|
851
|
-
const index =
|
|
1630
|
+
const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
|
|
852
1631
|
if (index === void 0) {
|
|
853
1632
|
throw new exports.QvdValidationError("A value is missing from the symbol table", {
|
|
854
1633
|
field: columns[column],
|
|
@@ -865,29 +1644,6 @@ var init_QvdFileWriter = __esm({
|
|
|
865
1644
|
}
|
|
866
1645
|
this._symbolIndexByValue = null;
|
|
867
1646
|
}
|
|
868
|
-
/**
|
|
869
|
-
* Converts a raw value/literal to a QVD symbol.
|
|
870
|
-
*
|
|
871
|
-
* @param {any} raw The raw value/literal to convert.
|
|
872
|
-
* @return {QvdSymbol|null} The converted QVD symbol.
|
|
873
|
-
*/
|
|
874
|
-
static _convertRawToSymbol(raw) {
|
|
875
|
-
if (raw === null || raw === void 0) {
|
|
876
|
-
return null;
|
|
877
|
-
}
|
|
878
|
-
const INT32_MIN = -2147483648;
|
|
879
|
-
const INT32_MAX = 2147483647;
|
|
880
|
-
const isInteger = typeof raw === "number" && Number.isInteger(raw);
|
|
881
|
-
const isFloat = typeof raw === "number" && !Number.isInteger(raw);
|
|
882
|
-
const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
|
|
883
|
-
if (isInteger && isWithinInt32Range) {
|
|
884
|
-
return exports.QvdSymbol.fromDualIntValue(raw, raw.toString());
|
|
885
|
-
} else if (isFloat || isInteger && !isWithinInt32Range) {
|
|
886
|
-
return exports.QvdSymbol.fromDualDoubleValue(raw, raw.toString());
|
|
887
|
-
} else {
|
|
888
|
-
return exports.QvdSymbol.fromStringValue(raw);
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
1647
|
/**
|
|
892
1648
|
* Persists the data frame to a QVD file.
|
|
893
1649
|
*/
|
|
@@ -957,7 +1713,7 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
957
1713
|
return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
|
|
958
1714
|
}
|
|
959
1715
|
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
|
|
960
|
-
const costOf = (rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0);
|
|
1716
|
+
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
961
1717
|
if (costOf(totalRows) <= budget) {
|
|
962
1718
|
return totalRows;
|
|
963
1719
|
}
|
|
@@ -978,11 +1734,11 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
|
|
|
978
1734
|
}
|
|
979
1735
|
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
|
|
980
1736
|
const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
|
|
981
|
-
const fits = (chunk) => {
|
|
1737
|
+
const fits = /* @__PURE__ */ __name((chunk) => {
|
|
982
1738
|
const live = Math.min(chunk * liveRowsPerChunk, covered);
|
|
983
1739
|
const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
|
|
984
1740
|
return cost <= budget;
|
|
985
|
-
};
|
|
1741
|
+
}, "fits");
|
|
986
1742
|
if (fits(covered)) {
|
|
987
1743
|
return covered;
|
|
988
1744
|
}
|
|
@@ -1072,9 +1828,9 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
1072
1828
|
if (nothingFits) {
|
|
1073
1829
|
advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
|
|
1074
1830
|
} else if (containerBound) {
|
|
1075
|
-
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${recommendedValue
|
|
1831
|
+
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
|
|
1076
1832
|
} else {
|
|
1077
|
-
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${recommendedValue
|
|
1833
|
+
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
1078
1834
|
}
|
|
1079
1835
|
throw new exports.QvdValidationError(
|
|
1080
1836
|
`Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
@@ -1102,6 +1858,9 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
1102
1858
|
);
|
|
1103
1859
|
}
|
|
1104
1860
|
}
|
|
1861
|
+
function formatCount(value) {
|
|
1862
|
+
return value.toLocaleString("en-US");
|
|
1863
|
+
}
|
|
1105
1864
|
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
1106
1865
|
const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
|
|
1107
1866
|
if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
@@ -1116,18 +1875,30 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
1116
1875
|
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
1117
1876
|
const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
|
|
1118
1877
|
console.warn(
|
|
1119
|
-
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${rowsToLoad
|
|
1878
|
+
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
1120
1879
|
);
|
|
1121
1880
|
}
|
|
1122
1881
|
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
1123
1882
|
var init_memoryUtils = __esm({
|
|
1124
1883
|
"src/util/memoryUtils.js"() {
|
|
1125
1884
|
init_QvdErrors();
|
|
1885
|
+
__name(getHeapLimit, "getHeapLimit");
|
|
1886
|
+
__name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
|
|
1887
|
+
__name(getMemoryBudget, "getMemoryBudget");
|
|
1126
1888
|
HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
|
|
1127
1889
|
MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
1890
|
+
__name(usableOldSpaceLimit, "usableOldSpaceLimit");
|
|
1128
1891
|
BASE_BYTES = 16 * 1024 * 1024;
|
|
1129
1892
|
ROW_BASE_BYTES = 72;
|
|
1130
1893
|
PER_CELL_BYTES = 8;
|
|
1894
|
+
__name(estimateExternalMemory, "estimateExternalMemory");
|
|
1895
|
+
__name(estimateRowMemory, "estimateRowMemory");
|
|
1896
|
+
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
1897
|
+
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
1898
|
+
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
1899
|
+
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
1900
|
+
__name(formatCount, "formatCount");
|
|
1901
|
+
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
1131
1902
|
}
|
|
1132
1903
|
});
|
|
1133
1904
|
|
|
@@ -1365,273 +2136,246 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
1365
2136
|
field: field["FieldName"],
|
|
1366
2137
|
bitOffset,
|
|
1367
2138
|
bitWidth,
|
|
1368
|
-
recordSizeInBits,
|
|
1369
|
-
file: filePath,
|
|
1370
|
-
stage: "parseIndexTable"
|
|
1371
|
-
});
|
|
1372
|
-
}
|
|
1373
|
-
}
|
|
1374
|
-
var init_validationUtils = __esm({
|
|
1375
|
-
"src/util/validationUtils.js"() {
|
|
1376
|
-
init_QvdErrors();
|
|
1377
|
-
init_memoryUtils();
|
|
1378
|
-
init_bitUtils();
|
|
1379
|
-
}
|
|
1380
|
-
});
|
|
1381
|
-
|
|
1382
|
-
// src/util/symbolParser.js
|
|
1383
|
-
function parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1384
|
-
if (pointer + 4 > bufferLength) {
|
|
1385
|
-
throw new exports.QvdCorruptedError("Buffer overflow reading integer symbol", {
|
|
1386
|
-
field: fieldName,
|
|
1387
|
-
pointer,
|
|
1388
|
-
bufferSize: bufferLength,
|
|
1389
|
-
file: filePath,
|
|
1390
|
-
stage: "parseSymbolTable"
|
|
1391
|
-
});
|
|
1392
|
-
}
|
|
1393
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1394
|
-
const value = Buffer.from(byteData).readIntLE(0, byteData.length);
|
|
1395
|
-
return { symbol: exports.QvdSymbol.fromIntValue(value), bytesRead: 4 };
|
|
1396
|
-
}
|
|
1397
|
-
function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1398
|
-
if (pointer + 8 > bufferLength) {
|
|
1399
|
-
throw new exports.QvdCorruptedError("Buffer overflow reading double symbol", {
|
|
1400
|
-
field: fieldName,
|
|
1401
|
-
pointer,
|
|
1402
|
-
bufferSize: bufferLength,
|
|
1403
|
-
file: filePath,
|
|
1404
|
-
stage: "parseSymbolTable"
|
|
1405
|
-
});
|
|
1406
|
-
}
|
|
1407
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1408
|
-
const value = Buffer.from(byteData).readDoubleLE(0);
|
|
1409
|
-
return { symbol: exports.QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
|
|
1410
|
-
}
|
|
1411
|
-
function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1412
|
-
const startPointer = pointer;
|
|
1413
|
-
const maxStringLength = 1048576;
|
|
1414
|
-
let stringLength = 0;
|
|
1415
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1416
|
-
if (stringLength >= maxStringLength) {
|
|
1417
|
-
throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1418
|
-
field: fieldName,
|
|
1419
|
-
maxLength: maxStringLength,
|
|
1420
|
-
file: filePath,
|
|
1421
|
-
stage: "parseSymbolTable"
|
|
1422
|
-
});
|
|
1423
|
-
}
|
|
1424
|
-
pointer++;
|
|
1425
|
-
stringLength++;
|
|
1426
|
-
}
|
|
1427
|
-
if (pointer >= bufferLength) {
|
|
1428
|
-
throw new exports.QvdCorruptedError("String symbol not null-terminated", {
|
|
1429
|
-
field: fieldName,
|
|
1430
|
-
pointer,
|
|
1431
|
-
bufferSize: bufferLength,
|
|
1432
|
-
file: filePath,
|
|
1433
|
-
stage: "parseSymbolTable"
|
|
1434
|
-
});
|
|
1435
|
-
}
|
|
1436
|
-
const value = symbolBuffer.subarray(startPointer, pointer).toString("utf-8");
|
|
1437
|
-
return { symbol: exports.QvdSymbol.fromStringValue(value), bytesRead: pointer - startPointer + 1 };
|
|
1438
|
-
}
|
|
1439
|
-
function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1440
|
-
const startPointer = pointer;
|
|
1441
|
-
const maxStringLength = 1048576;
|
|
1442
|
-
let stringLength = 0;
|
|
1443
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1444
|
-
if (stringLength >= maxStringLength) {
|
|
1445
|
-
throw new exports.QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1446
|
-
field: fieldName,
|
|
1447
|
-
maxLength: maxStringLength,
|
|
1448
|
-
file: filePath,
|
|
1449
|
-
stage: "parseSymbolTable"
|
|
1450
|
-
});
|
|
1451
|
-
}
|
|
1452
|
-
pointer++;
|
|
1453
|
-
stringLength++;
|
|
1454
|
-
}
|
|
1455
|
-
if (pointer >= bufferLength) {
|
|
1456
|
-
throw new exports.QvdCorruptedError("String symbol not null-terminated", {
|
|
1457
|
-
field: fieldName,
|
|
1458
|
-
pointer,
|
|
1459
|
-
bufferSize: bufferLength,
|
|
1460
|
-
file: filePath,
|
|
1461
|
-
stage: "parseSymbolTable"
|
|
1462
|
-
});
|
|
1463
|
-
}
|
|
1464
|
-
return pointer - startPointer + 1;
|
|
1465
|
-
}
|
|
1466
|
-
function parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1467
|
-
if (pointer + 4 > bufferLength) {
|
|
1468
|
-
throw new exports.QvdCorruptedError("Buffer overflow reading dual integer symbol", {
|
|
1469
|
-
field: fieldName,
|
|
1470
|
-
pointer,
|
|
1471
|
-
bufferSize: bufferLength,
|
|
1472
|
-
file: filePath,
|
|
1473
|
-
stage: "parseSymbolTable"
|
|
1474
|
-
});
|
|
1475
|
-
}
|
|
1476
|
-
const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1477
|
-
const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
|
|
1478
|
-
pointer += 4;
|
|
1479
|
-
const stringStart = pointer;
|
|
1480
|
-
const maxStringLength = 1048576;
|
|
1481
|
-
let stringLength = 0;
|
|
1482
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1483
|
-
if (stringLength >= maxStringLength) {
|
|
1484
|
-
throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1485
|
-
field: fieldName,
|
|
1486
|
-
maxLength: maxStringLength,
|
|
1487
|
-
file: filePath,
|
|
1488
|
-
stage: "parseSymbolTable"
|
|
1489
|
-
});
|
|
1490
|
-
}
|
|
1491
|
-
pointer++;
|
|
1492
|
-
stringLength++;
|
|
1493
|
-
}
|
|
1494
|
-
if (pointer >= bufferLength) {
|
|
1495
|
-
throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1496
|
-
field: fieldName,
|
|
1497
|
-
pointer,
|
|
1498
|
-
bufferSize: bufferLength,
|
|
1499
|
-
file: filePath,
|
|
1500
|
-
stage: "parseSymbolTable"
|
|
1501
|
-
});
|
|
1502
|
-
}
|
|
1503
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1504
|
-
return { symbol: exports.QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
|
|
1505
|
-
}
|
|
1506
|
-
function parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1507
|
-
if (pointer + 8 > bufferLength) {
|
|
1508
|
-
throw new exports.QvdCorruptedError("Buffer overflow reading dual double symbol", {
|
|
1509
|
-
field: fieldName,
|
|
1510
|
-
pointer,
|
|
1511
|
-
bufferSize: bufferLength,
|
|
1512
|
-
file: filePath,
|
|
1513
|
-
stage: "parseSymbolTable"
|
|
1514
|
-
});
|
|
1515
|
-
}
|
|
1516
|
-
const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1517
|
-
const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
|
|
1518
|
-
pointer += 8;
|
|
1519
|
-
const stringStart = pointer;
|
|
1520
|
-
const maxStringLength = 1048576;
|
|
1521
|
-
let stringLength = 0;
|
|
1522
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1523
|
-
if (stringLength >= maxStringLength) {
|
|
1524
|
-
throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1525
|
-
field: fieldName,
|
|
1526
|
-
maxLength: maxStringLength,
|
|
1527
|
-
file: filePath,
|
|
1528
|
-
stage: "parseSymbolTable"
|
|
1529
|
-
});
|
|
1530
|
-
}
|
|
1531
|
-
pointer++;
|
|
1532
|
-
stringLength++;
|
|
1533
|
-
}
|
|
1534
|
-
if (pointer >= bufferLength) {
|
|
1535
|
-
throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1536
|
-
field: fieldName,
|
|
1537
|
-
pointer,
|
|
1538
|
-
bufferSize: bufferLength,
|
|
2139
|
+
recordSizeInBits,
|
|
1539
2140
|
file: filePath,
|
|
1540
|
-
stage: "
|
|
2141
|
+
stage: "parseIndexTable"
|
|
1541
2142
|
});
|
|
1542
2143
|
}
|
|
1543
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1544
|
-
return {
|
|
1545
|
-
symbol: exports.QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
|
|
1546
|
-
bytesRead: pointer - (stringStart - 8) + 1
|
|
1547
|
-
};
|
|
1548
2144
|
}
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
2145
|
+
var init_validationUtils = __esm({
|
|
2146
|
+
"src/util/validationUtils.js"() {
|
|
2147
|
+
init_QvdErrors();
|
|
2148
|
+
init_memoryUtils();
|
|
2149
|
+
init_bitUtils();
|
|
2150
|
+
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2151
|
+
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2152
|
+
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2153
|
+
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2154
|
+
__name(validateRecordCount, "validateRecordCount");
|
|
2155
|
+
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2156
|
+
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
2157
|
+
}
|
|
2158
|
+
});
|
|
2159
|
+
|
|
2160
|
+
// src/util/symbolParser.js
|
|
2161
|
+
function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
|
|
2162
|
+
const bufferLength = symbolBuffer.length;
|
|
2163
|
+
const found = symbolBuffer.indexOf(0, from);
|
|
2164
|
+
if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
|
|
2165
|
+
throw new exports.QvdCorruptedError(`${kind} exceeds maximum length`, {
|
|
1552
2166
|
field: fieldName,
|
|
1553
|
-
|
|
1554
|
-
bufferSize: bufferLength,
|
|
2167
|
+
maxLength: MAX_TEXT_BYTES,
|
|
1555
2168
|
file: filePath,
|
|
1556
2169
|
stage: "parseSymbolTable"
|
|
1557
2170
|
});
|
|
1558
2171
|
}
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
const maxStringLength = 1048576;
|
|
1562
|
-
let stringLength = 0;
|
|
1563
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1564
|
-
if (stringLength >= maxStringLength) {
|
|
1565
|
-
throw new exports.QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1566
|
-
field: fieldName,
|
|
1567
|
-
maxLength: maxStringLength,
|
|
1568
|
-
file: filePath,
|
|
1569
|
-
stage: "parseSymbolTable"
|
|
1570
|
-
});
|
|
1571
|
-
}
|
|
1572
|
-
pointer++;
|
|
1573
|
-
stringLength++;
|
|
1574
|
-
}
|
|
1575
|
-
if (pointer >= bufferLength) {
|
|
1576
|
-
throw new exports.QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
2172
|
+
if (found === -1) {
|
|
2173
|
+
throw new exports.QvdCorruptedError(`${kind} not null-terminated`, {
|
|
1577
2174
|
field: fieldName,
|
|
1578
|
-
pointer,
|
|
2175
|
+
pointer: bufferLength,
|
|
1579
2176
|
bufferSize: bufferLength,
|
|
1580
2177
|
file: filePath,
|
|
1581
2178
|
stage: "parseSymbolTable"
|
|
1582
2179
|
});
|
|
1583
2180
|
}
|
|
1584
|
-
return
|
|
2181
|
+
return found;
|
|
2182
|
+
}
|
|
2183
|
+
function overflow(message, pointer, bufferLength, fieldName, filePath) {
|
|
2184
|
+
throw new exports.QvdCorruptedError(message, {
|
|
2185
|
+
field: fieldName,
|
|
2186
|
+
pointer,
|
|
2187
|
+
bufferSize: bufferLength,
|
|
2188
|
+
file: filePath,
|
|
2189
|
+
stage: "parseSymbolTable"
|
|
2190
|
+
});
|
|
1585
2191
|
}
|
|
1586
|
-
function
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
2192
|
+
function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
|
|
2193
|
+
const bufferLength = symbolBuffer.length;
|
|
2194
|
+
const numbers = [];
|
|
2195
|
+
const texts = [];
|
|
2196
|
+
let pointer = start;
|
|
2197
|
+
while (pointer < end) {
|
|
2198
|
+
const typeByte = symbolBuffer[pointer++];
|
|
2199
|
+
const decode = keep === null || keep.has(numbers.length);
|
|
2200
|
+
let number = null;
|
|
2201
|
+
let text = null;
|
|
2202
|
+
switch (typeByte) {
|
|
2203
|
+
case 1: {
|
|
2204
|
+
if (decode) {
|
|
2205
|
+
if (pointer + 4 > bufferLength) {
|
|
2206
|
+
overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
|
|
2207
|
+
}
|
|
2208
|
+
number = symbolBuffer.readInt32LE(pointer);
|
|
2209
|
+
}
|
|
2210
|
+
pointer += 4;
|
|
2211
|
+
break;
|
|
1591
2212
|
}
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
2213
|
+
case 2: {
|
|
2214
|
+
if (decode) {
|
|
2215
|
+
if (pointer + 8 > bufferLength) {
|
|
2216
|
+
overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
|
|
2217
|
+
}
|
|
2218
|
+
number = symbolBuffer.readDoubleLE(pointer);
|
|
2219
|
+
}
|
|
2220
|
+
pointer += 8;
|
|
2221
|
+
break;
|
|
1597
2222
|
}
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
2223
|
+
case 4: {
|
|
2224
|
+
const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
|
|
2225
|
+
if (decode) {
|
|
2226
|
+
text = symbolBuffer.toString("utf8", pointer, terminator);
|
|
2227
|
+
}
|
|
2228
|
+
pointer = terminator + 1;
|
|
2229
|
+
break;
|
|
1603
2230
|
}
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
2231
|
+
case 5:
|
|
2232
|
+
case 6: {
|
|
2233
|
+
const numberBytes = typeByte === 5 ? 4 : 8;
|
|
2234
|
+
if (pointer + numberBytes > bufferLength) {
|
|
2235
|
+
const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
|
|
2236
|
+
overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
|
|
2237
|
+
}
|
|
2238
|
+
const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
|
|
2239
|
+
if (decode) {
|
|
2240
|
+
number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
|
|
2241
|
+
text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
|
|
2242
|
+
}
|
|
2243
|
+
pointer = terminator + 1;
|
|
2244
|
+
break;
|
|
1610
2245
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2246
|
+
default: {
|
|
2247
|
+
throw new exports.QvdParseError("Unknown symbol type byte", {
|
|
2248
|
+
typeByte: typeByte.toString(16),
|
|
2249
|
+
offset: pointer - 1,
|
|
2250
|
+
file: filePath,
|
|
2251
|
+
stage: "parseSymbolTable"
|
|
2252
|
+
});
|
|
1617
2253
|
}
|
|
1618
|
-
const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
|
|
1619
|
-
return { symbol: null, bytesRead };
|
|
1620
|
-
}
|
|
1621
|
-
default: {
|
|
1622
|
-
throw new exports.QvdParseError("Unknown symbol type byte", {
|
|
1623
|
-
typeByte: typeByte.toString(16),
|
|
1624
|
-
offset: pointer - 1,
|
|
1625
|
-
file: filePath,
|
|
1626
|
-
stage: "parseSymbolTable"
|
|
1627
|
-
});
|
|
1628
2254
|
}
|
|
2255
|
+
numbers.push(number);
|
|
2256
|
+
texts.push(text);
|
|
1629
2257
|
}
|
|
2258
|
+
return { numbers, texts };
|
|
1630
2259
|
}
|
|
2260
|
+
var MAX_TEXT_BYTES;
|
|
1631
2261
|
var init_symbolParser = __esm({
|
|
1632
2262
|
"src/util/symbolParser.js"() {
|
|
1633
|
-
init_QvdSymbol();
|
|
1634
2263
|
init_QvdErrors();
|
|
2264
|
+
MAX_TEXT_BYTES = 1048576;
|
|
2265
|
+
__name(textEnd, "textEnd");
|
|
2266
|
+
__name(overflow, "overflow");
|
|
2267
|
+
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2268
|
+
}
|
|
2269
|
+
});
|
|
2270
|
+
|
|
2271
|
+
// src/util/resolveSymbols.js
|
|
2272
|
+
function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
|
|
2273
|
+
const length = symbols.numbers.length;
|
|
2274
|
+
const values = new Array(length);
|
|
2275
|
+
const entryValues = [];
|
|
2276
|
+
const numbers = [];
|
|
2277
|
+
const texts = [];
|
|
2278
|
+
let pure = 0;
|
|
2279
|
+
let partial = false;
|
|
2280
|
+
for (let index = 0; index < length; index++) {
|
|
2281
|
+
const text = symbols.texts[index];
|
|
2282
|
+
const number = symbols.numbers[index];
|
|
2283
|
+
if (text === null) {
|
|
2284
|
+
values[index] = number === null ? void 0 : number;
|
|
2285
|
+
if (number !== null) pure++;
|
|
2286
|
+
continue;
|
|
2287
|
+
}
|
|
2288
|
+
if (number === null) {
|
|
2289
|
+
if (coerce && isNumericText(text)) {
|
|
2290
|
+
values[index] = Number(text);
|
|
2291
|
+
entryValues.push(values[index]);
|
|
2292
|
+
numbers.push(null);
|
|
2293
|
+
texts.push(text);
|
|
2294
|
+
partial = true;
|
|
2295
|
+
} else {
|
|
2296
|
+
values[index] = text;
|
|
2297
|
+
pure++;
|
|
2298
|
+
}
|
|
2299
|
+
continue;
|
|
2300
|
+
}
|
|
2301
|
+
partial = true;
|
|
2302
|
+
if (mode === "both") {
|
|
2303
|
+
values[index] = dualFromSymbol(number, text);
|
|
2304
|
+
continue;
|
|
2305
|
+
}
|
|
2306
|
+
values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
|
|
2307
|
+
entryValues.push(values[index]);
|
|
2308
|
+
numbers.push(number);
|
|
2309
|
+
texts.push(text);
|
|
2310
|
+
}
|
|
2311
|
+
let entry = null;
|
|
2312
|
+
if (entryValues.length > 0) {
|
|
2313
|
+
entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
|
|
2314
|
+
field,
|
|
2315
|
+
values: Object.freeze(entryValues),
|
|
2316
|
+
numbers: Object.freeze(numbers),
|
|
2317
|
+
texts: Object.freeze(texts)
|
|
2318
|
+
});
|
|
2319
|
+
}
|
|
2320
|
+
return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
|
|
2321
|
+
}
|
|
2322
|
+
function collides(symbols, values, recorded) {
|
|
2323
|
+
for (let index = 0; index < values.length; index++) {
|
|
2324
|
+
if (!isPure(symbols, index, values[index])) {
|
|
2325
|
+
continue;
|
|
2326
|
+
}
|
|
2327
|
+
if (recorded.has(values[index])) {
|
|
2328
|
+
return true;
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
return false;
|
|
2332
|
+
}
|
|
2333
|
+
function isPure(symbols, index, value) {
|
|
2334
|
+
const text = symbols.texts[index];
|
|
2335
|
+
const number = symbols.numbers[index];
|
|
2336
|
+
return text === null ? number !== null : number === null && value === text;
|
|
2337
|
+
}
|
|
2338
|
+
function collisionEntry(symbols, field, values, mode) {
|
|
2339
|
+
const recorded = /* @__PURE__ */ new Set();
|
|
2340
|
+
const inPassA = values.map((value, index) => {
|
|
2341
|
+
if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
|
|
2342
|
+
return false;
|
|
2343
|
+
}
|
|
2344
|
+
const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
|
|
2345
|
+
if (recordedHere) {
|
|
2346
|
+
recorded.add(values[index]);
|
|
2347
|
+
}
|
|
2348
|
+
return recordedHere;
|
|
2349
|
+
});
|
|
2350
|
+
const entryValues = [];
|
|
2351
|
+
const numbers = [];
|
|
2352
|
+
const texts = [];
|
|
2353
|
+
for (let index = 0; index < values.length; index++) {
|
|
2354
|
+
if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
|
|
2355
|
+
entryValues.push(values[index]);
|
|
2356
|
+
numbers.push(symbols.numbers[index]);
|
|
2357
|
+
texts.push(symbols.texts[index]);
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2360
|
+
return Object.freeze({
|
|
2361
|
+
field,
|
|
2362
|
+
values: Object.freeze(entryValues),
|
|
2363
|
+
numbers: Object.freeze(numbers),
|
|
2364
|
+
texts: Object.freeze(texts)
|
|
2365
|
+
});
|
|
2366
|
+
}
|
|
2367
|
+
function symbolHalves(symbols) {
|
|
2368
|
+
return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
|
|
2369
|
+
}
|
|
2370
|
+
var init_resolveSymbols = __esm({
|
|
2371
|
+
"src/util/resolveSymbols.js"() {
|
|
2372
|
+
init_QvdDual();
|
|
2373
|
+
init_cellRules();
|
|
2374
|
+
__name(resolveFieldSymbols, "resolveFieldSymbols");
|
|
2375
|
+
__name(collides, "collides");
|
|
2376
|
+
__name(isPure, "isPure");
|
|
2377
|
+
__name(collisionEntry, "collisionEntry");
|
|
2378
|
+
__name(symbolHalves, "symbolHalves");
|
|
1635
2379
|
}
|
|
1636
2380
|
});
|
|
1637
2381
|
|
|
@@ -1641,21 +2385,49 @@ __export(QvdColumnTable_exports, {
|
|
|
1641
2385
|
QvdColumn: () => exports.QvdColumn,
|
|
1642
2386
|
QvdColumnTable: () => exports.QvdColumnTable
|
|
1643
2387
|
});
|
|
2388
|
+
function numberOf(value, number) {
|
|
2389
|
+
if (typeof value === "number") {
|
|
2390
|
+
return value;
|
|
2391
|
+
}
|
|
2392
|
+
if (typeof value === "string") {
|
|
2393
|
+
return number ?? NaN;
|
|
2394
|
+
}
|
|
2395
|
+
const dual = asDual(value);
|
|
2396
|
+
return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
|
|
2397
|
+
}
|
|
2398
|
+
function textOf(value) {
|
|
2399
|
+
if (typeof value === "string") {
|
|
2400
|
+
return value;
|
|
2401
|
+
}
|
|
2402
|
+
const dual = asDual(value);
|
|
2403
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
2404
|
+
}
|
|
1644
2405
|
exports.QvdColumn = void 0; exports.QvdColumnTable = void 0;
|
|
1645
2406
|
var init_QvdColumnTable = __esm({
|
|
1646
2407
|
"src/QvdColumnTable.js"() {
|
|
1647
2408
|
init_QvdErrors();
|
|
2409
|
+
init_cellRules();
|
|
1648
2410
|
init_readOptions();
|
|
2411
|
+
__name(numberOf, "numberOf");
|
|
2412
|
+
__name(textOf, "textOf");
|
|
1649
2413
|
exports.QvdColumn = class {
|
|
2414
|
+
static {
|
|
2415
|
+
__name(this, "QvdColumn");
|
|
2416
|
+
}
|
|
1650
2417
|
/**
|
|
1651
2418
|
* @param {string} name The field name.
|
|
1652
2419
|
* @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
|
|
1653
2420
|
* @param {Array<any>} symbols The field's distinct values, indexed by code.
|
|
2421
|
+
* @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
|
|
2422
|
+
* field whose values do not show them all - a dual read as one half, a string read as a number.
|
|
2423
|
+
* Without them, the halves are derived from the values: a string is its own text, a dual has its
|
|
2424
|
+
* own, and a number has none.
|
|
1654
2425
|
*/
|
|
1655
|
-
constructor(name, codes, symbols) {
|
|
2426
|
+
constructor(name, codes, symbols, halves = null) {
|
|
1656
2427
|
this._name = name;
|
|
1657
2428
|
this._codes = codes;
|
|
1658
2429
|
this._symbols = symbols;
|
|
2430
|
+
this._halves = halves;
|
|
1659
2431
|
Object.freeze(this);
|
|
1660
2432
|
}
|
|
1661
2433
|
/** @return {string} The field name. */
|
|
@@ -1682,6 +2454,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1682
2454
|
*
|
|
1683
2455
|
* One entry per distinct value, not per row: a few thousand entries for a column of millions.
|
|
1684
2456
|
*
|
|
2457
|
+
* A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
|
|
2458
|
+
* entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
|
|
2459
|
+
*
|
|
1685
2460
|
* @return {ReadonlyArray<any>} The dictionary.
|
|
1686
2461
|
*/
|
|
1687
2462
|
get symbols() {
|
|
@@ -1705,6 +2480,42 @@ var init_QvdColumnTable = __esm({
|
|
|
1705
2480
|
const code = this._codes[row];
|
|
1706
2481
|
return code < 0 ? null : this._symbols[code];
|
|
1707
2482
|
}
|
|
2483
|
+
/**
|
|
2484
|
+
* The text of one row: the text Qlik displays for its value.
|
|
2485
|
+
*
|
|
2486
|
+
* A string is its own text and a dual has its own. A value read as one half of a symbol - a date
|
|
2487
|
+
* read as its serial, a string read as a number - has the text the file stores for it, and a pure
|
|
2488
|
+
* number has none.
|
|
2489
|
+
*
|
|
2490
|
+
* @param {number} row The row index.
|
|
2491
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
2492
|
+
* @throws {QvdValidationError} If the row is not an integer within the column.
|
|
2493
|
+
*/
|
|
2494
|
+
textAt(row) {
|
|
2495
|
+
if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
|
|
2496
|
+
throw new exports.QvdValidationError("Row index out of bounds", {
|
|
2497
|
+
column: this._name,
|
|
2498
|
+
row,
|
|
2499
|
+
length: this._codes.length
|
|
2500
|
+
});
|
|
2501
|
+
}
|
|
2502
|
+
const code = this._codes[row];
|
|
2503
|
+
if (code < 0) {
|
|
2504
|
+
return null;
|
|
2505
|
+
}
|
|
2506
|
+
return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
|
|
2507
|
+
}
|
|
2508
|
+
/**
|
|
2509
|
+
* The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
|
|
2510
|
+
*
|
|
2511
|
+
* @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
|
|
2512
|
+
*/
|
|
2513
|
+
symbolTexts() {
|
|
2514
|
+
if (this._halves !== null) {
|
|
2515
|
+
return this._halves.texts;
|
|
2516
|
+
}
|
|
2517
|
+
return Object.freeze(this._symbols.map(textOf));
|
|
2518
|
+
}
|
|
1708
2519
|
/**
|
|
1709
2520
|
* Iterates the column's values without materialising it.
|
|
1710
2521
|
*
|
|
@@ -1748,6 +2559,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1748
2559
|
* one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
|
|
1749
2560
|
* codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
|
|
1750
2561
|
*
|
|
2562
|
+
* A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
|
|
2563
|
+
* `{duals: 'text'}` gives the serial the file stores for it, not NaN.
|
|
2564
|
+
*
|
|
1751
2565
|
* Scanning `codes` against this is the fastest way to read a column, because both sides are
|
|
1752
2566
|
* contiguous typed arrays and the dictionary fits in cache:
|
|
1753
2567
|
*
|
|
@@ -1769,8 +2583,7 @@ var init_QvdColumnTable = __esm({
|
|
|
1769
2583
|
numericSymbols() {
|
|
1770
2584
|
const out = new Float64Array(this._symbols.length);
|
|
1771
2585
|
for (let index = 0; index < this._symbols.length; index++) {
|
|
1772
|
-
|
|
1773
|
-
out[index] = typeof value === "number" ? value : NaN;
|
|
2586
|
+
out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
|
|
1774
2587
|
}
|
|
1775
2588
|
return out;
|
|
1776
2589
|
}
|
|
@@ -1785,7 +2598,8 @@ var init_QvdColumnTable = __esm({
|
|
|
1785
2598
|
* @param {Object} [options] Conversion options.
|
|
1786
2599
|
* @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
|
|
1787
2600
|
* number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
|
|
1788
|
-
* NaN, which is the right choice only when the caller knows the column is numeric.
|
|
2601
|
+
* NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
|
|
2602
|
+
* number here, as it is to `numericSymbols`.
|
|
1789
2603
|
* @return {Float64Array} One number per row.
|
|
1790
2604
|
* @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
|
|
1791
2605
|
*/
|
|
@@ -1805,6 +2619,11 @@ var init_QvdColumnTable = __esm({
|
|
|
1805
2619
|
out[row] = value;
|
|
1806
2620
|
continue;
|
|
1807
2621
|
}
|
|
2622
|
+
const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
|
|
2623
|
+
if (!Number.isNaN(number)) {
|
|
2624
|
+
out[row] = number;
|
|
2625
|
+
continue;
|
|
2626
|
+
}
|
|
1808
2627
|
if (onNonNumeric === "throw") {
|
|
1809
2628
|
throw new exports.QvdValidationError("Column holds a value that is not a number", {
|
|
1810
2629
|
column: this._name,
|
|
@@ -1820,21 +2639,30 @@ var init_QvdColumnTable = __esm({
|
|
|
1820
2639
|
}
|
|
1821
2640
|
};
|
|
1822
2641
|
exports.QvdColumnTable = class {
|
|
2642
|
+
static {
|
|
2643
|
+
__name(this, "QvdColumnTable");
|
|
2644
|
+
}
|
|
1823
2645
|
/**
|
|
1824
2646
|
* @param {Object} decoded What the reader decoded.
|
|
1825
2647
|
* @param {Array<string>} decoded.columns Field names, in file order.
|
|
1826
2648
|
* @param {Array<Int32Array>} decoded.codesByField One code array per field.
|
|
1827
2649
|
* @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
|
|
2650
|
+
* @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
|
|
2651
|
+
* or null for a field whose values show them.
|
|
1828
2652
|
* @param {number} decoded.rowCount Rows decoded.
|
|
1829
2653
|
* @param {any} decoded.metadata The raw QvdTableHeader.
|
|
2654
|
+
* @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
|
|
2655
|
+
* stored-symbol record, as a data frame of the same read carries it.
|
|
1830
2656
|
* @param {any} decoded.loadStats Statistics about the read.
|
|
1831
2657
|
*/
|
|
1832
|
-
constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
|
|
2658
|
+
constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
|
|
1833
2659
|
this._columns = columns;
|
|
1834
2660
|
this._codesByField = codesByField;
|
|
1835
2661
|
this._symbolsByField = symbolsByField;
|
|
2662
|
+
this._halvesByField = halvesByField ?? null;
|
|
1836
2663
|
this._rowCount = rowCount;
|
|
1837
2664
|
this._metadata = metadata;
|
|
2665
|
+
this._storedSymbols = storedSymbols ?? null;
|
|
1838
2666
|
this._loadStats = loadStats;
|
|
1839
2667
|
}
|
|
1840
2668
|
/**
|
|
@@ -1853,6 +2681,13 @@ var init_QvdColumnTable = __esm({
|
|
|
1853
2681
|
* @param {number} [options.offset] File row to start at.
|
|
1854
2682
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
1855
2683
|
* appear. Unselected fields have their symbols skipped entirely.
|
|
2684
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
|
|
2685
|
+
* number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
|
|
2686
|
+
* text and `numericSymbols` the number. Anything else throws.
|
|
2687
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
|
|
2688
|
+
* number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
2689
|
+
* `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
|
|
2690
|
+
* Anything but a boolean throws.
|
|
1856
2691
|
* @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
|
|
1857
2692
|
* @param {AbortSignal} [options.signal] Cancels the read.
|
|
1858
2693
|
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
@@ -1892,6 +2727,14 @@ var init_QvdColumnTable = __esm({
|
|
|
1892
2727
|
get loadStats() {
|
|
1893
2728
|
return this._loadStats;
|
|
1894
2729
|
}
|
|
2730
|
+
/**
|
|
2731
|
+
* The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
|
|
2732
|
+
*
|
|
2733
|
+
* @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
|
|
2734
|
+
*/
|
|
2735
|
+
get storedSymbols() {
|
|
2736
|
+
return this._storedSymbols;
|
|
2737
|
+
}
|
|
1895
2738
|
/**
|
|
1896
2739
|
* One column.
|
|
1897
2740
|
*
|
|
@@ -1907,7 +2750,12 @@ var init_QvdColumnTable = __esm({
|
|
|
1907
2750
|
availableColumns: this._columns
|
|
1908
2751
|
});
|
|
1909
2752
|
}
|
|
1910
|
-
return new exports.QvdColumn(
|
|
2753
|
+
return new exports.QvdColumn(
|
|
2754
|
+
name,
|
|
2755
|
+
this._codesByField[index],
|
|
2756
|
+
this._symbolsByField[index],
|
|
2757
|
+
this._halvesByField?.[index] ?? null
|
|
2758
|
+
);
|
|
1911
2759
|
}
|
|
1912
2760
|
};
|
|
1913
2761
|
}
|
|
@@ -1918,6 +2766,16 @@ var QvdFileReader_exports = {};
|
|
|
1918
2766
|
__export(QvdFileReader_exports, {
|
|
1919
2767
|
QvdFileReader: () => exports.QvdFileReader
|
|
1920
2768
|
});
|
|
2769
|
+
function closeReadStream(stream) {
|
|
2770
|
+
if (stream.closed) {
|
|
2771
|
+
return Promise.resolve();
|
|
2772
|
+
}
|
|
2773
|
+
return new Promise((resolve) => {
|
|
2774
|
+
stream.once("close", () => resolve());
|
|
2775
|
+
stream.once("error", () => resolve());
|
|
2776
|
+
stream.destroy();
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
1921
2779
|
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS; exports.QvdFileReader = void 0;
|
|
1922
2780
|
var init_QvdFileReader = __esm({
|
|
1923
2781
|
"src/QvdFileReader.js"() {
|
|
@@ -1929,10 +2787,16 @@ var init_QvdFileReader = __esm({
|
|
|
1929
2787
|
init_validationUtils();
|
|
1930
2788
|
init_symbolParser();
|
|
1931
2789
|
init_readOptions();
|
|
2790
|
+
init_resolveSymbols();
|
|
2791
|
+
init_storedSymbols();
|
|
1932
2792
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
1933
2793
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
1934
2794
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
2795
|
+
__name(closeReadStream, "closeReadStream");
|
|
1935
2796
|
exports.QvdFileReader = class {
|
|
2797
|
+
static {
|
|
2798
|
+
__name(this, "QvdFileReader");
|
|
2799
|
+
}
|
|
1936
2800
|
/**
|
|
1937
2801
|
* Constructs a new QVD file parser.
|
|
1938
2802
|
*
|
|
@@ -1959,6 +2823,18 @@ var init_QvdFileReader = __esm({
|
|
|
1959
2823
|
* smaller files, raise it to keep the simpler single-pass read for longer.
|
|
1960
2824
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
1961
2825
|
* appear. Null reads every field, in file order. An unknown or repeated name is refused.
|
|
2826
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
2827
|
+
* text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
|
|
2828
|
+
* value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
|
|
2829
|
+
* `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
|
|
2830
|
+
* `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
|
|
2831
|
+
* the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
|
|
2832
|
+
* throws a `QvdValidationError`.
|
|
2833
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
2834
|
+
* reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
|
|
2835
|
+
* every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
|
|
2836
|
+
* dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
|
|
2837
|
+
* dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
|
|
1962
2838
|
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
1963
2839
|
* read proceeds - the same shape `QvdFileWriter` emits.
|
|
1964
2840
|
* @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
|
|
@@ -1971,11 +2847,15 @@ var init_QvdFileReader = __esm({
|
|
|
1971
2847
|
symbolFilteringThreshold = 50 * 1024 * 1024,
|
|
1972
2848
|
materialisesRows = true,
|
|
1973
2849
|
fields = null,
|
|
2850
|
+
duals,
|
|
2851
|
+
coerceNumericStrings,
|
|
1974
2852
|
onProgress,
|
|
1975
2853
|
signal
|
|
1976
2854
|
} = options;
|
|
1977
2855
|
this._materialisesRows = materialisesRows;
|
|
1978
2856
|
this._path = validatePath(filePath, allowedDir);
|
|
2857
|
+
this._duals = normaliseDuals(duals, this._path);
|
|
2858
|
+
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
1979
2859
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
1980
2860
|
this._symbolFilteringThreshold = symbolFilteringThreshold;
|
|
1981
2861
|
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
@@ -2047,15 +2927,17 @@ var init_QvdFileReader = __esm({
|
|
|
2047
2927
|
/**
|
|
2048
2928
|
* Reads the binary data of the QVD file.
|
|
2049
2929
|
*
|
|
2050
|
-
*
|
|
2051
|
-
*
|
|
2930
|
+
* A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
|
|
2931
|
+
* needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
|
|
2932
|
+
* over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
|
|
2052
2933
|
*
|
|
2053
|
-
*
|
|
2054
|
-
*
|
|
2055
|
-
*
|
|
2056
|
-
*
|
|
2934
|
+
* The saving is in the index table and the rows, not in the symbol table, which is read in
|
|
2935
|
+
* full whatever the window because a stored index in any row can address any symbol. So the
|
|
2936
|
+
* gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
|
|
2937
|
+
* values there is very little to save, which is what `symbolFilteringThreshold` and the
|
|
2938
|
+
* two-pass path exist for.
|
|
2057
2939
|
*
|
|
2058
|
-
* Algorithm for
|
|
2940
|
+
* Algorithm for a windowed read:
|
|
2059
2941
|
* 1. Stream-read the file until XML header delimiter is found
|
|
2060
2942
|
* 2. Parse header to determine symbol table and index table locations
|
|
2061
2943
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
@@ -2126,6 +3008,8 @@ var init_QvdFileReader = __esm({
|
|
|
2126
3008
|
if (!isExpectedEarlyClose) {
|
|
2127
3009
|
throw error;
|
|
2128
3010
|
}
|
|
3011
|
+
} finally {
|
|
3012
|
+
await closeReadStream(stream);
|
|
2129
3013
|
}
|
|
2130
3014
|
if (headerDelimiterIndex === -1) {
|
|
2131
3015
|
throw new exports.QvdCorruptedError(
|
|
@@ -2500,27 +3384,16 @@ var init_QvdFileReader = __esm({
|
|
|
2500
3384
|
this._throwIfAborted();
|
|
2501
3385
|
const symbolsOffset = parseInt(field["Offset"], 10);
|
|
2502
3386
|
const symbolsLength = parseInt(field["Length"], 10);
|
|
2503
|
-
const
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
symbolBuffer,
|
|
2514
|
-
pointer,
|
|
2515
|
-
symbolBuffer.length,
|
|
2516
|
-
fieldName,
|
|
2517
|
-
this._path,
|
|
2518
|
-
shouldKeepSymbol
|
|
2519
|
-
);
|
|
2520
|
-
symbols.push(symbol);
|
|
2521
|
-
pointer += bytesRead - 1;
|
|
2522
|
-
symbolIndex++;
|
|
2523
|
-
}
|
|
3387
|
+
const symbols = parseFieldSymbols(
|
|
3388
|
+
symbolBuffer,
|
|
3389
|
+
symbolsOffset,
|
|
3390
|
+
symbolsOffset + symbolsLength,
|
|
3391
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
3392
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
3393
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
3394
|
+
field["FieldName"],
|
|
3395
|
+
this._path
|
|
3396
|
+
);
|
|
2524
3397
|
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
2525
3398
|
return symbols;
|
|
2526
3399
|
});
|
|
@@ -2631,10 +3504,16 @@ var init_QvdFileReader = __esm({
|
|
|
2631
3504
|
const prepared = await this._prepare(rows);
|
|
2632
3505
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
2633
3506
|
const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
|
|
2634
|
-
return new exports.QvdDataFrame(
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
3507
|
+
return new exports.QvdDataFrame(
|
|
3508
|
+
data,
|
|
3509
|
+
prepared.columns,
|
|
3510
|
+
prepared.metadata,
|
|
3511
|
+
{
|
|
3512
|
+
...prepared.loadStats,
|
|
3513
|
+
rowsLoaded: data.length
|
|
3514
|
+
},
|
|
3515
|
+
prepared.storedSymbols
|
|
3516
|
+
);
|
|
2638
3517
|
}
|
|
2639
3518
|
/**
|
|
2640
3519
|
* Reads the file as columns, without ever materialising rows.
|
|
@@ -2652,7 +3531,7 @@ var init_QvdFileReader = __esm({
|
|
|
2652
3531
|
*/
|
|
2653
3532
|
async loadColumnar(window = null) {
|
|
2654
3533
|
const rows = normaliseWindow(window, this._path);
|
|
2655
|
-
const prepared = await this._prepare(rows);
|
|
3534
|
+
const prepared = await this._prepare(rows, null, true);
|
|
2656
3535
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
2657
3536
|
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
2658
3537
|
assert2__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
@@ -2660,8 +3539,10 @@ var init_QvdFileReader = __esm({
|
|
|
2660
3539
|
columns: prepared.columns,
|
|
2661
3540
|
codesByField: this._indexColumns,
|
|
2662
3541
|
symbolsByField: prepared.resolvedByField,
|
|
3542
|
+
halvesByField: prepared.halvesByField,
|
|
2663
3543
|
rowCount: this._rowsDecoded,
|
|
2664
3544
|
metadata: prepared.metadata,
|
|
3545
|
+
storedSymbols: prepared.storedSymbols,
|
|
2665
3546
|
loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
|
|
2666
3547
|
});
|
|
2667
3548
|
}
|
|
@@ -2706,11 +3587,17 @@ var init_QvdFileReader = __esm({
|
|
|
2706
3587
|
const offset = prepared.offset + done;
|
|
2707
3588
|
await this._parseIndexTable({ offset, limit: count });
|
|
2708
3589
|
const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
|
|
2709
|
-
yield new exports.QvdDataFrame(
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
3590
|
+
yield new exports.QvdDataFrame(
|
|
3591
|
+
data,
|
|
3592
|
+
prepared.columns,
|
|
3593
|
+
prepared.metadata,
|
|
3594
|
+
{
|
|
3595
|
+
...prepared.loadStats,
|
|
3596
|
+
offset,
|
|
3597
|
+
rowsLoaded: data.length
|
|
3598
|
+
},
|
|
3599
|
+
prepared.storedSymbols
|
|
3600
|
+
);
|
|
2714
3601
|
}
|
|
2715
3602
|
}
|
|
2716
3603
|
/**
|
|
@@ -2727,12 +3614,17 @@ var init_QvdFileReader = __esm({
|
|
|
2727
3614
|
* @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
|
|
2728
3615
|
* is fewer than the window covers, and how many of them one row of the caller's chunk size
|
|
2729
3616
|
* accounts for. Only `iterateRows` passes it; every other read holds what it covers.
|
|
3617
|
+
* @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
|
|
3618
|
+
* cells do not show them, which only a columnar read has a use for.
|
|
2730
3619
|
* @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
|
|
2731
|
-
* resolvedByField: Array<Array<any>>,
|
|
2732
|
-
*
|
|
3620
|
+
* resolvedByField: Array<Array<any>>,
|
|
3621
|
+
* halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
|
|
3622
|
+
* storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
|
|
3623
|
+
* rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
|
|
3624
|
+
* against it.
|
|
2733
3625
|
* @private
|
|
2734
3626
|
*/
|
|
2735
|
-
async _prepare(window, liveRows = null) {
|
|
3627
|
+
async _prepare(window, liveRows = null, wantHalves = false) {
|
|
2736
3628
|
this._throwIfAborted();
|
|
2737
3629
|
await this._readData(window, false, liveRows);
|
|
2738
3630
|
this._emitProgress("header", 0, 1);
|
|
@@ -2755,17 +3647,31 @@ var init_QvdFileReader = __esm({
|
|
|
2755
3647
|
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
2756
3648
|
assert2__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
2757
3649
|
this._throwIfAborted();
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
3650
|
+
assert2__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3651
|
+
const resolvedByField = [];
|
|
3652
|
+
const halvesByField = [];
|
|
3653
|
+
const entries = [];
|
|
3654
|
+
this._symbolTable.forEach((symbols, position) => {
|
|
3655
|
+
const { values, entry, halves } = resolveFieldSymbols(
|
|
3656
|
+
symbols,
|
|
3657
|
+
// @ts-ignore - asserted above
|
|
3658
|
+
this._selectedFields[position]["FieldName"],
|
|
3659
|
+
this._duals,
|
|
3660
|
+
this._coerceNumericStrings,
|
|
3661
|
+
wantHalves
|
|
3662
|
+
);
|
|
3663
|
+
resolvedByField.push(values);
|
|
3664
|
+
halvesByField.push(halves);
|
|
3665
|
+
if (entry !== null) {
|
|
3666
|
+
entries.push(entry);
|
|
2763
3667
|
}
|
|
2764
|
-
return resolved2;
|
|
2765
3668
|
});
|
|
2766
|
-
assert2__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
2767
3669
|
const columns = this._selectedFields.map((field) => field["FieldName"]);
|
|
2768
3670
|
const metadata = this._header["QvdTableHeader"];
|
|
3671
|
+
const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
|
|
3672
|
+
if (storedSymbols !== null) {
|
|
3673
|
+
attachStoredSymbols(metadata, storedSymbols);
|
|
3674
|
+
}
|
|
2769
3675
|
const loadStats = {
|
|
2770
3676
|
symbolTableBytes: symbolTableLength,
|
|
2771
3677
|
totalRows,
|
|
@@ -2774,7 +3680,16 @@ var init_QvdFileReader = __esm({
|
|
|
2774
3680
|
symbolFiltering: symbolsToKeep !== null,
|
|
2775
3681
|
symbolsKept
|
|
2776
3682
|
};
|
|
2777
|
-
return {
|
|
3683
|
+
return {
|
|
3684
|
+
columns,
|
|
3685
|
+
metadata,
|
|
3686
|
+
loadStats,
|
|
3687
|
+
resolvedByField,
|
|
3688
|
+
halvesByField,
|
|
3689
|
+
storedSymbols,
|
|
3690
|
+
rowsAvailable,
|
|
3691
|
+
offset: resolved.offset
|
|
3692
|
+
};
|
|
2778
3693
|
}
|
|
2779
3694
|
/**
|
|
2780
3695
|
* Builds rows from the columns currently decoded.
|
|
@@ -2818,24 +3733,95 @@ var init_QvdFileReader = __esm({
|
|
|
2818
3733
|
});
|
|
2819
3734
|
|
|
2820
3735
|
// src/QvdDataFrame.js
|
|
3736
|
+
function defaultFieldHeader(fieldName) {
|
|
3737
|
+
return {
|
|
3738
|
+
FieldName: fieldName,
|
|
3739
|
+
BitOffset: 0,
|
|
3740
|
+
BitWidth: 0,
|
|
3741
|
+
Bias: 0,
|
|
3742
|
+
NoOfSymbols: 0,
|
|
3743
|
+
Offset: 0,
|
|
3744
|
+
Length: 0,
|
|
3745
|
+
Comment: "",
|
|
3746
|
+
NumberFormat: {
|
|
3747
|
+
Type: "UNKNOWN",
|
|
3748
|
+
nDec: "0",
|
|
3749
|
+
UseThou: "0",
|
|
3750
|
+
Fmt: "",
|
|
3751
|
+
Dec: "",
|
|
3752
|
+
Thou: ""
|
|
3753
|
+
},
|
|
3754
|
+
Tags: {}
|
|
3755
|
+
};
|
|
3756
|
+
}
|
|
3757
|
+
function defaultHeader(columns) {
|
|
3758
|
+
return {
|
|
3759
|
+
QvBuildNo: 50667,
|
|
3760
|
+
CreatorDoc: "",
|
|
3761
|
+
CreateUtcTime: "",
|
|
3762
|
+
SourceCreateUtcTime: "",
|
|
3763
|
+
SourceFileUtcTime: "",
|
|
3764
|
+
SourceFileSize: -1,
|
|
3765
|
+
StaleUtcTime: "",
|
|
3766
|
+
TableName: "",
|
|
3767
|
+
Fields: {
|
|
3768
|
+
QvdFieldHeader: columns.map(defaultFieldHeader)
|
|
3769
|
+
},
|
|
3770
|
+
NoOfRecords: 0,
|
|
3771
|
+
RecordByteSize: 0,
|
|
3772
|
+
Offset: 0,
|
|
3773
|
+
Length: 0,
|
|
3774
|
+
Compression: "",
|
|
3775
|
+
Comment: "",
|
|
3776
|
+
EncryptionInfo: "",
|
|
3777
|
+
TableTags: "",
|
|
3778
|
+
ProfilingData: "",
|
|
3779
|
+
Lineage: {}
|
|
3780
|
+
};
|
|
3781
|
+
}
|
|
2821
3782
|
exports.QvdDataFrame = void 0;
|
|
2822
3783
|
var init_QvdDataFrame = __esm({
|
|
2823
3784
|
"src/QvdDataFrame.js"() {
|
|
2824
3785
|
init_QvdErrors();
|
|
3786
|
+
init_cellRules();
|
|
2825
3787
|
init_readOptions();
|
|
3788
|
+
init_storedSymbols();
|
|
3789
|
+
__name(defaultFieldHeader, "defaultFieldHeader");
|
|
3790
|
+
__name(defaultHeader, "defaultHeader");
|
|
2826
3791
|
exports.QvdDataFrame = class _QvdDataFrame {
|
|
3792
|
+
static {
|
|
3793
|
+
__name(this, "QvdDataFrame");
|
|
3794
|
+
}
|
|
2827
3795
|
/**
|
|
2828
3796
|
* Represents the data frame stored inside a QVD file.
|
|
3797
|
+
*
|
|
3798
|
+
* The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
|
|
3799
|
+
* its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
|
|
3800
|
+
* Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
|
|
3801
|
+
* holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
|
|
3802
|
+
* header's record did for a frame built from some of a read's columns.
|
|
3803
|
+
*
|
|
2829
3804
|
* @param {Array<Array<any>>} data The data of the data frame.
|
|
2830
3805
|
* @param {Array<string>} columns The columns of the data frame.
|
|
2831
3806
|
* @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
|
|
2832
3807
|
* @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
|
|
3808
|
+
* @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
|
|
3809
|
+
* shows only one half of its symbol (optional) - see `storedSymbols`.
|
|
3810
|
+
* @throws {QvdValidationError} If the record is malformed.
|
|
2833
3811
|
*/
|
|
2834
|
-
constructor(data, columns, metadata = null, loadStats = null) {
|
|
3812
|
+
constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
|
|
2835
3813
|
this._data = data;
|
|
2836
3814
|
this._columns = columns;
|
|
2837
3815
|
this._metadata = metadata;
|
|
3816
|
+
this._ownsMetadata = false;
|
|
2838
3817
|
this._loadStats = loadStats;
|
|
3818
|
+
this._storedSymbols = narrowStoredSymbols(
|
|
3819
|
+
normaliseStoredSymbols(
|
|
3820
|
+
// @ts-ignore - a symbol-keyed property the reader defines on the header object
|
|
3821
|
+
storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
|
|
3822
|
+
),
|
|
3823
|
+
columns
|
|
3824
|
+
);
|
|
2839
3825
|
}
|
|
2840
3826
|
/**
|
|
2841
3827
|
* Returns the data of the data frame.
|
|
@@ -2862,6 +3848,28 @@ var init_QvdDataFrame = __esm({
|
|
|
2862
3848
|
get metadata() {
|
|
2863
3849
|
return this._metadata;
|
|
2864
3850
|
}
|
|
3851
|
+
/**
|
|
3852
|
+
* What the frame's cells were read from, where a cell shows only one half of its symbol.
|
|
3853
|
+
*
|
|
3854
|
+
* A dual read as its number has a text the cell does not show; one read as its text has a number;
|
|
3855
|
+
* a string read as a number has the text it was spelled with. The record keeps those halves, per
|
|
3856
|
+
* field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
|
|
3857
|
+
* and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
|
|
3858
|
+
* `select`, `toDict` and `fromDict`.
|
|
3859
|
+
*
|
|
3860
|
+
* Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
|
|
3861
|
+
* the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
|
|
3862
|
+
* pure number.
|
|
3863
|
+
*
|
|
3864
|
+
* @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
|
|
3865
|
+
* showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
|
|
3866
|
+
* in the record it was given or found - one from `select`, or one built from some of a read's
|
|
3867
|
+
* columns and its header - has an empty record rather than null, because a frame given null takes
|
|
3868
|
+
* the record its header carries, which describes every field of the read.
|
|
3869
|
+
*/
|
|
3870
|
+
get storedSymbols() {
|
|
3871
|
+
return this._storedSymbols;
|
|
3872
|
+
}
|
|
2865
3873
|
/**
|
|
2866
3874
|
* Returns statistics about the read that produced this data frame.
|
|
2867
3875
|
*
|
|
@@ -2982,54 +3990,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2982
3990
|
* @property {string} [profilingData] - Profiling data
|
|
2983
3991
|
* @property {Object|string} [lineage] - Lineage
|
|
2984
3992
|
*/
|
|
3993
|
+
/**
|
|
3994
|
+
* The header a metadata setter may change: this frame's own.
|
|
3995
|
+
*
|
|
3996
|
+
* A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
|
|
3997
|
+
* `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
|
|
3998
|
+
* change copies it, and a change made through one frame never reaches another. A frame with no header
|
|
3999
|
+
* gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
|
|
4000
|
+
* so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
|
|
4001
|
+
*
|
|
4002
|
+
* @return {any} The header.
|
|
4003
|
+
*/
|
|
4004
|
+
_ownMetadata() {
|
|
4005
|
+
if (!this._metadata) {
|
|
4006
|
+
this._metadata = defaultHeader(this._columns);
|
|
4007
|
+
} else if (!this._ownsMetadata) {
|
|
4008
|
+
const record = this._metadata[STORED_SYMBOLS];
|
|
4009
|
+
this._metadata = structuredClone(this._metadata);
|
|
4010
|
+
if (record) {
|
|
4011
|
+
attachStoredSymbols(this._metadata, record);
|
|
4012
|
+
}
|
|
4013
|
+
}
|
|
4014
|
+
this._ownsMetadata = true;
|
|
4015
|
+
return this._metadata;
|
|
4016
|
+
}
|
|
2985
4017
|
/**
|
|
2986
4018
|
* Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
|
|
4019
|
+
*
|
|
4020
|
+
* The change applies to this frame only, never to a frame it was derived from or shares a header
|
|
4021
|
+
* with.
|
|
4022
|
+
*
|
|
2987
4023
|
* @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
|
|
2988
4024
|
*/
|
|
2989
4025
|
setFileMetadata(metadata) {
|
|
2990
|
-
|
|
2991
|
-
this._metadata = {
|
|
2992
|
-
QvBuildNo: 50667,
|
|
2993
|
-
CreatorDoc: "",
|
|
2994
|
-
CreateUtcTime: "",
|
|
2995
|
-
SourceCreateUtcTime: "",
|
|
2996
|
-
SourceFileUtcTime: "",
|
|
2997
|
-
SourceFileSize: -1,
|
|
2998
|
-
StaleUtcTime: "",
|
|
2999
|
-
TableName: "",
|
|
3000
|
-
Fields: {
|
|
3001
|
-
QvdFieldHeader: this._columns.map((column) => ({
|
|
3002
|
-
FieldName: column,
|
|
3003
|
-
BitOffset: 0,
|
|
3004
|
-
BitWidth: 0,
|
|
3005
|
-
Bias: 0,
|
|
3006
|
-
NoOfSymbols: 0,
|
|
3007
|
-
Offset: 0,
|
|
3008
|
-
Length: 0,
|
|
3009
|
-
Comment: "",
|
|
3010
|
-
NumberFormat: {
|
|
3011
|
-
Type: "UNKNOWN",
|
|
3012
|
-
nDec: "0",
|
|
3013
|
-
UseThou: "0",
|
|
3014
|
-
Fmt: "",
|
|
3015
|
-
Dec: "",
|
|
3016
|
-
Thou: ""
|
|
3017
|
-
},
|
|
3018
|
-
Tags: {}
|
|
3019
|
-
}))
|
|
3020
|
-
},
|
|
3021
|
-
NoOfRecords: 0,
|
|
3022
|
-
RecordByteSize: 0,
|
|
3023
|
-
Offset: 0,
|
|
3024
|
-
Length: 0,
|
|
3025
|
-
Compression: "",
|
|
3026
|
-
Comment: "",
|
|
3027
|
-
EncryptionInfo: "",
|
|
3028
|
-
TableTags: "",
|
|
3029
|
-
ProfilingData: "",
|
|
3030
|
-
Lineage: {}
|
|
3031
|
-
};
|
|
3032
|
-
}
|
|
4026
|
+
const header = this._ownMetadata();
|
|
3033
4027
|
const modifiableFields = [
|
|
3034
4028
|
"qvBuildNo",
|
|
3035
4029
|
"creatorDoc",
|
|
@@ -3064,7 +4058,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3064
4058
|
};
|
|
3065
4059
|
modifiableFields.forEach((field) => {
|
|
3066
4060
|
if (metadata[field] !== void 0) {
|
|
3067
|
-
|
|
4061
|
+
header[fieldMapping[field]] = metadata[field];
|
|
3068
4062
|
}
|
|
3069
4063
|
});
|
|
3070
4064
|
}
|
|
@@ -3077,30 +4071,44 @@ var init_QvdDataFrame = __esm({
|
|
|
3077
4071
|
/**
|
|
3078
4072
|
* Sets modifiable field-level metadata for a specific field.
|
|
3079
4073
|
* Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
|
|
4074
|
+
*
|
|
4075
|
+
* Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
|
|
4076
|
+
* header does not describe. The change applies to this frame only, never to a frame it was derived
|
|
4077
|
+
* from or shares a header with.
|
|
4078
|
+
*
|
|
3080
4079
|
* @param {string} fieldName The name of the field.
|
|
3081
4080
|
* @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
|
|
4081
|
+
* @throws {QvdValidationError} If the frame has no column of that name.
|
|
3082
4082
|
*/
|
|
3083
4083
|
setFieldMetadata(fieldName, metadata) {
|
|
3084
|
-
if (!this.
|
|
3085
|
-
|
|
4084
|
+
if (!this._columns.includes(fieldName)) {
|
|
4085
|
+
throw new exports.QvdValidationError(`Column '${fieldName}' does not exist`, {
|
|
4086
|
+
column: fieldName,
|
|
4087
|
+
availableColumns: this._columns
|
|
4088
|
+
});
|
|
3086
4089
|
}
|
|
3087
|
-
|
|
4090
|
+
const header = this._ownMetadata();
|
|
4091
|
+
if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
|
|
4092
|
+
header.Fields = { QvdFieldHeader: [] };
|
|
4093
|
+
}
|
|
4094
|
+
let fields = header.Fields.QvdFieldHeader;
|
|
3088
4095
|
if (!Array.isArray(fields)) {
|
|
3089
4096
|
fields = [fields];
|
|
3090
|
-
|
|
4097
|
+
header.Fields.QvdFieldHeader = fields;
|
|
3091
4098
|
}
|
|
3092
|
-
|
|
3093
|
-
if (
|
|
3094
|
-
|
|
4099
|
+
let field = fields.find((f) => f.FieldName === fieldName);
|
|
4100
|
+
if (!field) {
|
|
4101
|
+
field = defaultFieldHeader(fieldName);
|
|
4102
|
+
fields.push(field);
|
|
3095
4103
|
}
|
|
3096
4104
|
if (metadata.comment !== void 0) {
|
|
3097
|
-
|
|
4105
|
+
field.Comment = metadata.comment;
|
|
3098
4106
|
}
|
|
3099
4107
|
if (metadata.numberFormat !== void 0) {
|
|
3100
|
-
|
|
4108
|
+
field.NumberFormat = metadata.numberFormat;
|
|
3101
4109
|
}
|
|
3102
4110
|
if (metadata.tags !== void 0) {
|
|
3103
|
-
|
|
4111
|
+
field.Tags = metadata.tags;
|
|
3104
4112
|
}
|
|
3105
4113
|
}
|
|
3106
4114
|
/**
|
|
@@ -3117,7 +4125,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3117
4125
|
type: typeof n
|
|
3118
4126
|
});
|
|
3119
4127
|
}
|
|
3120
|
-
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
|
|
4128
|
+
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
|
|
3121
4129
|
}
|
|
3122
4130
|
/**
|
|
3123
4131
|
* Returns the last n rows of the data frame.
|
|
@@ -3133,7 +4141,13 @@ var init_QvdDataFrame = __esm({
|
|
|
3133
4141
|
type: typeof n
|
|
3134
4142
|
});
|
|
3135
4143
|
}
|
|
3136
|
-
return new _QvdDataFrame(
|
|
4144
|
+
return new _QvdDataFrame(
|
|
4145
|
+
n === 0 ? [] : this._data.slice(-n),
|
|
4146
|
+
this._columns,
|
|
4147
|
+
this._metadata,
|
|
4148
|
+
null,
|
|
4149
|
+
this._storedSymbols
|
|
4150
|
+
);
|
|
3137
4151
|
}
|
|
3138
4152
|
/**
|
|
3139
4153
|
* Returns the selected rows of the data frame.
|
|
@@ -3161,7 +4175,9 @@ var init_QvdDataFrame = __esm({
|
|
|
3161
4175
|
return new _QvdDataFrame(
|
|
3162
4176
|
args.map((index) => this._data[index]),
|
|
3163
4177
|
this._columns,
|
|
3164
|
-
this._metadata
|
|
4178
|
+
this._metadata,
|
|
4179
|
+
null,
|
|
4180
|
+
this._storedSymbols
|
|
3165
4181
|
);
|
|
3166
4182
|
}
|
|
3167
4183
|
/**
|
|
@@ -3173,6 +4189,50 @@ var init_QvdDataFrame = __esm({
|
|
|
3173
4189
|
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
3174
4190
|
*/
|
|
3175
4191
|
at(row, column) {
|
|
4192
|
+
const index = this._cellIndex(row, column);
|
|
4193
|
+
return this._data[row][index];
|
|
4194
|
+
}
|
|
4195
|
+
/**
|
|
4196
|
+
* Returns the text of the value at the specified row and column.
|
|
4197
|
+
*
|
|
4198
|
+
* The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
|
|
4199
|
+
* `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
|
|
4200
|
+
* or the string it was spelled as - and is null for a number that was stored as a pure number.
|
|
4201
|
+
*
|
|
4202
|
+
* ```js
|
|
4203
|
+
* const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
|
|
4204
|
+
* df.at(0, 'date'); // -52593
|
|
4205
|
+
* df.textAt(0, 'date'); // '1756-01-01'
|
|
4206
|
+
* ```
|
|
4207
|
+
*
|
|
4208
|
+
* @param {number} row The index of the row.
|
|
4209
|
+
* @param {string} column The name of the column.
|
|
4210
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
4211
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4212
|
+
*/
|
|
4213
|
+
textAt(row, column) {
|
|
4214
|
+
const index = this._cellIndex(row, column);
|
|
4215
|
+
const value = this._data[row][index];
|
|
4216
|
+
if (typeof value === "string") {
|
|
4217
|
+
return value;
|
|
4218
|
+
}
|
|
4219
|
+
if (typeof value === "number") {
|
|
4220
|
+
const entry = storedSymbolsEntry(this._storedSymbols, column);
|
|
4221
|
+
return entry === null ? null : storedTextOf(entry, value);
|
|
4222
|
+
}
|
|
4223
|
+
const dual = asDual(value);
|
|
4224
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
4225
|
+
}
|
|
4226
|
+
/**
|
|
4227
|
+
* Checks a row and a column name, and returns the column's position.
|
|
4228
|
+
*
|
|
4229
|
+
* @param {number} row The index of the row.
|
|
4230
|
+
* @param {string} column The name of the column.
|
|
4231
|
+
* @return {number} The column's position.
|
|
4232
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4233
|
+
* @private
|
|
4234
|
+
*/
|
|
4235
|
+
_cellIndex(row, column) {
|
|
3176
4236
|
if (typeof row !== "number" || !Number.isInteger(row)) {
|
|
3177
4237
|
throw new exports.QvdValidationError("Row index must be an integer", {
|
|
3178
4238
|
provided: row,
|
|
@@ -3192,7 +4252,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3192
4252
|
availableColumns: this._columns
|
|
3193
4253
|
});
|
|
3194
4254
|
}
|
|
3195
|
-
return this.
|
|
4255
|
+
return this._columns.indexOf(column);
|
|
3196
4256
|
}
|
|
3197
4257
|
/**
|
|
3198
4258
|
* Selects the specified columns from the data frame.
|
|
@@ -3213,15 +4273,34 @@ var init_QvdDataFrame = __esm({
|
|
|
3213
4273
|
const indices = args.map((arg) => this._columns.indexOf(arg));
|
|
3214
4274
|
const data = this._data.map((row) => indices.map((index) => row[index]));
|
|
3215
4275
|
const columns = indices.map((index) => this._columns[index]);
|
|
3216
|
-
return new _QvdDataFrame(data, columns, this._metadata);
|
|
4276
|
+
return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
|
|
3217
4277
|
}
|
|
3218
4278
|
/**
|
|
3219
4279
|
* Returns the data frame as a dictionary.
|
|
3220
4280
|
*
|
|
3221
|
-
*
|
|
4281
|
+
* Everything a frame needs to write the same file again, as plain data: the header, and the
|
|
4282
|
+
* stored-symbol record that says what cells showing one half of a symbol were read from. So
|
|
4283
|
+
* `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
|
|
4284
|
+
* `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
|
|
4285
|
+
*
|
|
4286
|
+
* @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
|
|
3222
4287
|
*/
|
|
3223
4288
|
async toDict() {
|
|
3224
|
-
return
|
|
4289
|
+
return this.toJSON();
|
|
4290
|
+
}
|
|
4291
|
+
/**
|
|
4292
|
+
* The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
|
|
4293
|
+
* `fromDict(JSON.parse(...))` can revive.
|
|
4294
|
+
*
|
|
4295
|
+
* @return {QvdDataFrameDict} The data frame as a dictionary.
|
|
4296
|
+
*/
|
|
4297
|
+
toJSON() {
|
|
4298
|
+
return {
|
|
4299
|
+
columns: this._columns,
|
|
4300
|
+
data: this._data,
|
|
4301
|
+
metadata: this._metadata,
|
|
4302
|
+
storedSymbols: this._storedSymbols
|
|
4303
|
+
};
|
|
3225
4304
|
}
|
|
3226
4305
|
/**
|
|
3227
4306
|
* Persists the data frame to a QVD file.
|
|
@@ -3258,6 +4337,21 @@ var init_QvdDataFrame = __esm({
|
|
|
3258
4337
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
3259
4338
|
* appear in the result. Unselected fields have their symbols skipped entirely rather than
|
|
3260
4339
|
* parsed and discarded. An unknown or repeated name throws.
|
|
4340
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
4341
|
+
* text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
|
|
4342
|
+
* `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
|
|
4343
|
+
* `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
|
|
4344
|
+
* implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
|
|
4345
|
+
* `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
|
|
4346
|
+
* double, a string and NULL read the same in every mode: a number, a string and null. Anything else
|
|
4347
|
+
* throws.
|
|
4348
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4349
|
+
* reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
|
|
4350
|
+
* reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
|
|
4351
|
+
* `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
|
|
4352
|
+
* `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
|
|
4353
|
+
* original string or dual back, and a value that two stored values read as is refused there
|
|
4354
|
+
* rather than written as either. Anything but a boolean throws.
|
|
3261
4355
|
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
3262
4356
|
* read proceeds - the same shape `toQvd`'s callback receives.
|
|
3263
4357
|
* @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
|
|
@@ -3275,7 +4369,8 @@ var init_QvdDataFrame = __esm({
|
|
|
3275
4369
|
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
|
|
3276
4370
|
* a lazy load switches to the two-pass filtering path. Defaults to 50MB.
|
|
3277
4371
|
* @throws {QvdValidationError} If a window option is not a non-negative integer, if both
|
|
3278
|
-
* `maxRows` and `limit` are given,
|
|
4372
|
+
* `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
|
|
4373
|
+
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
3279
4374
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
3280
4375
|
*/
|
|
3281
4376
|
static async fromQvd(path3, options = {}) {
|
|
@@ -3301,9 +4396,30 @@ var init_QvdDataFrame = __esm({
|
|
|
3301
4396
|
* A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
|
|
3302
4397
|
* run its body.
|
|
3303
4398
|
*
|
|
4399
|
+
* Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
|
|
4400
|
+
* them, so a chunk written on its own writes the symbols its cells were read from.
|
|
4401
|
+
*
|
|
3304
4402
|
* @param {string} path The path to the QVD file.
|
|
3305
|
-
* @param {Object} [options]
|
|
4403
|
+
* @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
|
|
3306
4404
|
* @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
|
|
4405
|
+
* @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
|
|
4406
|
+
* @param {number|null} [options.limit] Rows to cover, counting from `offset`.
|
|
4407
|
+
* @param {number} [options.offset=0] File row to start at.
|
|
4408
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
|
|
4409
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
|
|
4410
|
+
* its text, or a frozen `QvdDual` holding both. Anything else throws.
|
|
4411
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4412
|
+
* reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
4413
|
+
* `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
|
|
4414
|
+
* `storedSymbols`. Anything but a boolean throws.
|
|
4415
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
|
|
4416
|
+
* over the rows counts the whole window, not each chunk.
|
|
4417
|
+
* @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
|
|
4418
|
+
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
4419
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
|
|
4420
|
+
* charged for two chunks of rows rather than the window. Zero disables the check.
|
|
4421
|
+
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
|
|
4422
|
+
* windowed read switches to two-pass filtering.
|
|
3307
4423
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
3308
4424
|
*/
|
|
3309
4425
|
static async *iterate(path3, options = {}) {
|
|
@@ -3346,8 +4462,15 @@ var init_QvdDataFrame = __esm({
|
|
|
3346
4462
|
/**
|
|
3347
4463
|
* Constructs a data frame from a dictionary.
|
|
3348
4464
|
*
|
|
3349
|
-
*
|
|
4465
|
+
* Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
|
|
4466
|
+
* rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
|
|
4467
|
+
* spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
|
|
4468
|
+
* here rather than written.
|
|
4469
|
+
*
|
|
4470
|
+
* @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
|
|
3350
4471
|
* @return {Promise<QvdDataFrame>} The constructed data frame.
|
|
4472
|
+
* @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
|
|
4473
|
+
* or `storedSymbols` is malformed or names a field that is not one of the columns.
|
|
3351
4474
|
*/
|
|
3352
4475
|
static async fromDict(data) {
|
|
3353
4476
|
if (!data.columns) {
|
|
@@ -3360,18 +4483,292 @@ var init_QvdDataFrame = __esm({
|
|
|
3360
4483
|
data
|
|
3361
4484
|
});
|
|
3362
4485
|
}
|
|
3363
|
-
|
|
4486
|
+
const { metadata = null, storedSymbols = null } = data;
|
|
4487
|
+
if (metadata !== null && !isPlainObject(metadata)) {
|
|
4488
|
+
throw new exports.QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
|
|
4489
|
+
type: typeof metadata
|
|
4490
|
+
});
|
|
4491
|
+
}
|
|
4492
|
+
const record = normaliseStoredSymbols(storedSymbols);
|
|
4493
|
+
if (record !== null) {
|
|
4494
|
+
const unknown = record.find((entry) => !data.columns.includes(entry.field));
|
|
4495
|
+
if (unknown !== void 0) {
|
|
4496
|
+
throw new exports.QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
|
|
4497
|
+
field: unknown.field,
|
|
4498
|
+
availableColumns: data.columns
|
|
4499
|
+
});
|
|
4500
|
+
}
|
|
4501
|
+
}
|
|
4502
|
+
return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
|
|
3364
4503
|
}
|
|
3365
4504
|
};
|
|
3366
4505
|
}
|
|
3367
4506
|
});
|
|
3368
4507
|
|
|
4508
|
+
// src/QvdSymbol.js
|
|
4509
|
+
init_QvdErrors();
|
|
4510
|
+
init_cellRules();
|
|
4511
|
+
init_symbolBytes();
|
|
4512
|
+
function checkInteger(value, context) {
|
|
4513
|
+
if (typeof value === "number" && isStoredAsInt(value)) {
|
|
4514
|
+
return;
|
|
4515
|
+
}
|
|
4516
|
+
throw new exports.QvdValidationError(
|
|
4517
|
+
`The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
|
|
4518
|
+
{ ...context, half: "integer", type: typeof value }
|
|
4519
|
+
);
|
|
4520
|
+
}
|
|
4521
|
+
__name(checkInteger, "checkInteger");
|
|
4522
|
+
var QvdSymbol = class _QvdSymbol {
|
|
4523
|
+
static {
|
|
4524
|
+
__name(this, "QvdSymbol");
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Constructs a new QVD symbol.
|
|
4528
|
+
*
|
|
4529
|
+
* @param {number|null} intValue The integer value.
|
|
4530
|
+
* @param {number|null} doubleValue The double value.
|
|
4531
|
+
* @param {string|null} stringValue The string value.
|
|
4532
|
+
*/
|
|
4533
|
+
constructor(intValue, doubleValue, stringValue) {
|
|
4534
|
+
this._intValue = intValue;
|
|
4535
|
+
this._doubleValue = doubleValue;
|
|
4536
|
+
this._stringValue = stringValue;
|
|
4537
|
+
}
|
|
4538
|
+
/**
|
|
4539
|
+
* Returns the integer value of this symbol.
|
|
4540
|
+
*
|
|
4541
|
+
* @return {number|null} The integer value.
|
|
4542
|
+
*/
|
|
4543
|
+
get intValue() {
|
|
4544
|
+
return this._intValue;
|
|
4545
|
+
}
|
|
4546
|
+
/**
|
|
4547
|
+
* Returns the double value of this symbol.
|
|
4548
|
+
*
|
|
4549
|
+
* @return {number|null} The double value.
|
|
4550
|
+
*/
|
|
4551
|
+
get doubleValue() {
|
|
4552
|
+
return this._doubleValue;
|
|
4553
|
+
}
|
|
4554
|
+
/**
|
|
4555
|
+
* Returns the string value of this symbol.
|
|
4556
|
+
*
|
|
4557
|
+
* @return {string|null} The string value.
|
|
4558
|
+
*/
|
|
4559
|
+
get stringValue() {
|
|
4560
|
+
return this._stringValue;
|
|
4561
|
+
}
|
|
4562
|
+
/**
|
|
4563
|
+
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
4564
|
+
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
4565
|
+
*
|
|
4566
|
+
* @return {number|string|null} The primary value.
|
|
4567
|
+
*/
|
|
4568
|
+
toPrimaryValue() {
|
|
4569
|
+
if (null != this._stringValue) {
|
|
4570
|
+
return this._stringValue;
|
|
4571
|
+
} else if (null != this._intValue) {
|
|
4572
|
+
return this._intValue;
|
|
4573
|
+
} else if (null != this._doubleValue) {
|
|
4574
|
+
return this._doubleValue;
|
|
4575
|
+
} else {
|
|
4576
|
+
return null;
|
|
4577
|
+
}
|
|
4578
|
+
}
|
|
4579
|
+
/**
|
|
4580
|
+
* Converts the symbol to its byte representation.
|
|
4581
|
+
*
|
|
4582
|
+
* The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
|
|
4583
|
+
* double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
|
|
4584
|
+
* checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
|
|
4585
|
+
* outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
|
|
4586
|
+
* integer and a double silently lost the double, a text with a NUL produced a symbol that ends
|
|
4587
|
+
* early, and a text with an unpaired surrogate was written with U+FFFD in its place.
|
|
4588
|
+
*
|
|
4589
|
+
* A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
|
|
4590
|
+
* `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
|
|
4591
|
+
* `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
|
|
4592
|
+
*
|
|
4593
|
+
* @return {Buffer} The byte representation of the symbol.
|
|
4594
|
+
* @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
|
|
4595
|
+
* holds a half no symbol can store. The message names the half.
|
|
4596
|
+
*/
|
|
4597
|
+
toByteRepresentation() {
|
|
4598
|
+
const intValue = this._intValue ?? null;
|
|
4599
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4600
|
+
const stringValue = this._stringValue ?? null;
|
|
4601
|
+
if (intValue !== null && doubleValue !== null) {
|
|
4602
|
+
throw new exports.QvdValidationError("A symbol holds an integer or a double, not both", {
|
|
4603
|
+
intValue,
|
|
4604
|
+
doubleValue
|
|
4605
|
+
});
|
|
4606
|
+
}
|
|
4607
|
+
if (intValue === null && doubleValue === null && stringValue === null) {
|
|
4608
|
+
throw new exports.QvdValidationError("The symbol does not contain any value.", {
|
|
4609
|
+
intValue,
|
|
4610
|
+
doubleValue,
|
|
4611
|
+
stringValue
|
|
4612
|
+
});
|
|
4613
|
+
}
|
|
4614
|
+
if (intValue !== null) {
|
|
4615
|
+
checkInteger(intValue, {});
|
|
4616
|
+
}
|
|
4617
|
+
if (doubleValue !== null) {
|
|
4618
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4619
|
+
}
|
|
4620
|
+
if (stringValue !== null) {
|
|
4621
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4622
|
+
}
|
|
4623
|
+
const number = intValue ?? doubleValue;
|
|
4624
|
+
const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
|
|
4625
|
+
const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
|
|
4626
|
+
writeSymbol(buffer, 0, kind, number, stringValue);
|
|
4627
|
+
return buffer;
|
|
4628
|
+
}
|
|
4629
|
+
/**
|
|
4630
|
+
* Checks if this symbol is equal to another symbol.
|
|
4631
|
+
*
|
|
4632
|
+
* By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
|
|
4633
|
+
* `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
|
|
4634
|
+
* answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
|
|
4635
|
+
* or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
|
|
4636
|
+
* storage kind, so either kind matches.
|
|
4637
|
+
*
|
|
4638
|
+
* @param {*} value The object to compare with.
|
|
4639
|
+
* @return {boolean} True if the objects are equal, false otherwise.
|
|
4640
|
+
*/
|
|
4641
|
+
equals(value) {
|
|
4642
|
+
if (value === null || typeof value !== "object") {
|
|
4643
|
+
return false;
|
|
4644
|
+
}
|
|
4645
|
+
const intValue = this._intValue ?? null;
|
|
4646
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4647
|
+
const stringValue = this._stringValue ?? null;
|
|
4648
|
+
const dual = asDual(value);
|
|
4649
|
+
if (dual !== null) {
|
|
4650
|
+
return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
|
|
4651
|
+
}
|
|
4652
|
+
if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
|
|
4653
|
+
return false;
|
|
4654
|
+
}
|
|
4655
|
+
return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
|
|
4656
|
+
}
|
|
4657
|
+
/**
|
|
4658
|
+
* Constructs a pure integer value symbol.
|
|
4659
|
+
*
|
|
4660
|
+
* @param {number} intValue The integer value.
|
|
4661
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4662
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
|
|
4663
|
+
*/
|
|
4664
|
+
static fromIntValue(intValue) {
|
|
4665
|
+
checkInteger(intValue, {});
|
|
4666
|
+
return new _QvdSymbol(intValue, null, null);
|
|
4667
|
+
}
|
|
4668
|
+
/**
|
|
4669
|
+
* Constructs a pure double value symbol.
|
|
4670
|
+
*
|
|
4671
|
+
* @param {number} doubleValue The double value.
|
|
4672
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4673
|
+
* @throws {QvdValidationError} If the double is not a finite number.
|
|
4674
|
+
*/
|
|
4675
|
+
static fromDoubleValue(doubleValue) {
|
|
4676
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4677
|
+
return new _QvdSymbol(null, doubleValue, null);
|
|
4678
|
+
}
|
|
4679
|
+
/**
|
|
4680
|
+
* Constructs a pure string value symbol.
|
|
4681
|
+
*
|
|
4682
|
+
* @param {string} stringValue The string value.
|
|
4683
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4684
|
+
* @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
|
|
4685
|
+
*/
|
|
4686
|
+
static fromStringValue(stringValue) {
|
|
4687
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4688
|
+
return new _QvdSymbol(null, null, stringValue);
|
|
4689
|
+
}
|
|
4690
|
+
/**
|
|
4691
|
+
* Constructs a dual value symbol from an integer and a string value.
|
|
4692
|
+
*
|
|
4693
|
+
* @param {number} intValue The integer value.
|
|
4694
|
+
* @param {string} stringValue The string value.
|
|
4695
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4696
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
|
|
4697
|
+
* is not a string, or holds a NUL or an unpaired surrogate.
|
|
4698
|
+
*/
|
|
4699
|
+
static fromDualIntValue(intValue, stringValue) {
|
|
4700
|
+
checkInteger(intValue, {});
|
|
4701
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4702
|
+
return new _QvdSymbol(intValue, null, stringValue);
|
|
4703
|
+
}
|
|
4704
|
+
/**
|
|
4705
|
+
* Constructs a dual value symbol from a double and a string value.
|
|
4706
|
+
*
|
|
4707
|
+
* @param {number} doubleValue The double value.
|
|
4708
|
+
* @param {string} stringValue The string value.
|
|
4709
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4710
|
+
* @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
|
|
4711
|
+
* or holds a NUL or an unpaired surrogate.
|
|
4712
|
+
*/
|
|
4713
|
+
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
4714
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4715
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4716
|
+
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
4717
|
+
}
|
|
4718
|
+
};
|
|
4719
|
+
|
|
4720
|
+
// src/index.js
|
|
4721
|
+
init_QvdDual();
|
|
4722
|
+
|
|
4723
|
+
// src/util/qlikDate.js
|
|
4724
|
+
init_QvdErrors();
|
|
4725
|
+
init_cellRules();
|
|
4726
|
+
var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
|
|
4727
|
+
var MS_PER_DAY = 864e5;
|
|
4728
|
+
var MAX_DATE_MS = 864e13;
|
|
4729
|
+
var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4730
|
+
var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4731
|
+
function qlikSerialToDate(serial) {
|
|
4732
|
+
const dual = asDual(serial);
|
|
4733
|
+
const number = dual === null ? serial : dual.number;
|
|
4734
|
+
if (typeof number !== "number" || !Number.isFinite(number)) {
|
|
4735
|
+
throw new exports.QvdValidationError("A Qlik date serial must be a finite number", {
|
|
4736
|
+
provided: describeType(number),
|
|
4737
|
+
type: typeof serial
|
|
4738
|
+
});
|
|
4739
|
+
}
|
|
4740
|
+
const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
|
|
4741
|
+
if (!(Math.abs(ms) <= MAX_DATE_MS)) {
|
|
4742
|
+
throw new exports.QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
|
|
4743
|
+
serial: number,
|
|
4744
|
+
minSerial: MIN_SERIAL,
|
|
4745
|
+
maxSerial: MAX_SERIAL
|
|
4746
|
+
});
|
|
4747
|
+
}
|
|
4748
|
+
return new Date(ms);
|
|
4749
|
+
}
|
|
4750
|
+
__name(qlikSerialToDate, "qlikSerialToDate");
|
|
4751
|
+
function dateToQlikSerial(date) {
|
|
4752
|
+
const ms = util.types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
|
|
4753
|
+
if (Number.isNaN(ms)) {
|
|
4754
|
+
throw new exports.QvdValidationError("dateToQlikSerial needs a valid Date", {
|
|
4755
|
+
provided: describeType(date),
|
|
4756
|
+
type: typeof date
|
|
4757
|
+
});
|
|
4758
|
+
}
|
|
4759
|
+
return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4760
|
+
}
|
|
4761
|
+
__name(dateToQlikSerial, "dateToQlikSerial");
|
|
4762
|
+
|
|
3369
4763
|
// src/index.js
|
|
3370
|
-
init_QvdSymbol();
|
|
3371
4764
|
init_QvdDataFrame();
|
|
3372
4765
|
init_QvdColumnTable();
|
|
3373
4766
|
init_QvdFileReader();
|
|
3374
4767
|
init_QvdFileWriter();
|
|
3375
4768
|
init_QvdErrors();
|
|
4769
|
+
|
|
4770
|
+
exports.QvdSymbol = QvdSymbol;
|
|
4771
|
+
exports.dateToQlikSerial = dateToQlikSerial;
|
|
4772
|
+
exports.qlikSerialToDate = qlikSerialToDate;
|
|
3376
4773
|
//# sourceMappingURL=index.cjs.map
|
|
3377
4774
|
//# sourceMappingURL=index.cjs.map
|