qvdjs 0.11.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -1318
- package/dist/index.cjs +1969 -572
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1967 -574
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -5,9 +5,11 @@ import xml2 from 'xml2js';
|
|
|
5
5
|
import assert2 from 'assert';
|
|
6
6
|
import os from 'os';
|
|
7
7
|
import v8 from 'v8';
|
|
8
|
+
import { types } from 'util';
|
|
8
9
|
|
|
9
10
|
var __defProp = Object.defineProperty;
|
|
10
11
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
12
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
11
13
|
var __esm = (fn, res) => function __init() {
|
|
12
14
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
13
15
|
};
|
|
@@ -17,10 +19,14 @@ var __export = (target, all) => {
|
|
|
17
19
|
};
|
|
18
20
|
|
|
19
21
|
// src/QvdErrors.js
|
|
20
|
-
var QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
|
|
22
|
+
var ERROR_NAMES, QvdError, QvdParseError, QvdValidationError, QvdIOError, QvdCorruptedError, QvdSecurityError;
|
|
21
23
|
var init_QvdErrors = __esm({
|
|
22
24
|
"src/QvdErrors.js"() {
|
|
25
|
+
ERROR_NAMES = /* @__PURE__ */ new Map();
|
|
23
26
|
QvdError = class extends Error {
|
|
27
|
+
static {
|
|
28
|
+
__name(this, "QvdError");
|
|
29
|
+
}
|
|
24
30
|
/**
|
|
25
31
|
* Constructs a new QVD error.
|
|
26
32
|
*
|
|
@@ -30,13 +36,16 @@ var init_QvdErrors = __esm({
|
|
|
30
36
|
*/
|
|
31
37
|
constructor(message, code, context = {}) {
|
|
32
38
|
super(message);
|
|
33
|
-
this.name =
|
|
39
|
+
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
34
40
|
this.code = code;
|
|
35
41
|
this.context = context;
|
|
36
42
|
Error.captureStackTrace(this, this.constructor);
|
|
37
43
|
}
|
|
38
44
|
};
|
|
39
45
|
QvdParseError = class extends QvdError {
|
|
46
|
+
static {
|
|
47
|
+
__name(this, "QvdParseError");
|
|
48
|
+
}
|
|
40
49
|
/**
|
|
41
50
|
* Constructs a new QVD parse error.
|
|
42
51
|
*
|
|
@@ -48,6 +57,9 @@ var init_QvdErrors = __esm({
|
|
|
48
57
|
}
|
|
49
58
|
};
|
|
50
59
|
QvdValidationError = class extends QvdError {
|
|
60
|
+
static {
|
|
61
|
+
__name(this, "QvdValidationError");
|
|
62
|
+
}
|
|
51
63
|
/**
|
|
52
64
|
* Constructs a new QVD validation error.
|
|
53
65
|
*
|
|
@@ -59,6 +71,9 @@ var init_QvdErrors = __esm({
|
|
|
59
71
|
}
|
|
60
72
|
};
|
|
61
73
|
QvdIOError = class extends QvdError {
|
|
74
|
+
static {
|
|
75
|
+
__name(this, "QvdIOError");
|
|
76
|
+
}
|
|
62
77
|
/**
|
|
63
78
|
* Constructs a new QVD IO error.
|
|
64
79
|
*
|
|
@@ -70,6 +85,9 @@ var init_QvdErrors = __esm({
|
|
|
70
85
|
}
|
|
71
86
|
};
|
|
72
87
|
QvdCorruptedError = class extends QvdError {
|
|
88
|
+
static {
|
|
89
|
+
__name(this, "QvdCorruptedError");
|
|
90
|
+
}
|
|
73
91
|
/**
|
|
74
92
|
* Constructs a new QVD corrupted error.
|
|
75
93
|
*
|
|
@@ -81,6 +99,9 @@ var init_QvdErrors = __esm({
|
|
|
81
99
|
}
|
|
82
100
|
};
|
|
83
101
|
QvdSecurityError = class extends QvdError {
|
|
102
|
+
static {
|
|
103
|
+
__name(this, "QvdSecurityError");
|
|
104
|
+
}
|
|
84
105
|
/**
|
|
85
106
|
* Constructs a new QVD security error.
|
|
86
107
|
*
|
|
@@ -91,163 +112,298 @@ var init_QvdErrors = __esm({
|
|
|
91
112
|
super(message, "QVD_SECURITY_ERROR", context);
|
|
92
113
|
}
|
|
93
114
|
};
|
|
115
|
+
ERROR_NAMES.set(QvdError, "QvdError");
|
|
116
|
+
ERROR_NAMES.set(QvdParseError, "QvdParseError");
|
|
117
|
+
ERROR_NAMES.set(QvdValidationError, "QvdValidationError");
|
|
118
|
+
ERROR_NAMES.set(QvdIOError, "QvdIOError");
|
|
119
|
+
ERROR_NAMES.set(QvdCorruptedError, "QvdCorruptedError");
|
|
120
|
+
ERROR_NAMES.set(QvdSecurityError, "QvdSecurityError");
|
|
94
121
|
}
|
|
95
122
|
});
|
|
96
123
|
|
|
97
|
-
// src/
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
124
|
+
// src/util/cellRules.js
|
|
125
|
+
function asDual(value) {
|
|
126
|
+
if (value === null || typeof value !== "object") {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
if (value[DUAL_BRAND] === true) {
|
|
131
|
+
return value;
|
|
132
|
+
}
|
|
133
|
+
if (!isPlainObject(value)) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
const keys = Object.keys(value);
|
|
137
|
+
return keys.length === 2 && (keys[0] === "number" && keys[1] === "text" || keys[0] === "text" && keys[1] === "number") ? value : null;
|
|
138
|
+
} catch {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function isPlainObject(value) {
|
|
143
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
const prototype = Object.getPrototypeOf(value);
|
|
147
|
+
return prototype === null || Object.getPrototypeOf(prototype) === null;
|
|
148
|
+
}
|
|
149
|
+
function isNumericText(text) {
|
|
150
|
+
return text.trim() !== "" && Number.isFinite(Number(text));
|
|
151
|
+
}
|
|
152
|
+
function isStoredAsInt(value) {
|
|
153
|
+
return Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX;
|
|
154
|
+
}
|
|
155
|
+
function numberProblem(value) {
|
|
156
|
+
if (typeof value !== "number") {
|
|
157
|
+
return `is ${describeType(value)}, not a number`;
|
|
158
|
+
}
|
|
159
|
+
return Number.isFinite(value) ? null : `is ${String(value)}`;
|
|
160
|
+
}
|
|
161
|
+
function textProblem(value) {
|
|
162
|
+
if (typeof value !== "string") {
|
|
163
|
+
return { reason: "type" };
|
|
164
|
+
}
|
|
165
|
+
const position = value.indexOf(NUL);
|
|
166
|
+
if (position !== -1) {
|
|
167
|
+
return { reason: "nul", position };
|
|
168
|
+
}
|
|
169
|
+
return value.isWellFormed() ? null : { reason: "surrogate", position: unpairedSurrogateIndex(value) };
|
|
170
|
+
}
|
|
171
|
+
function unpairedSurrogateIndex(value) {
|
|
172
|
+
for (let index = 0; index < value.length; index++) {
|
|
173
|
+
const unit = value.charCodeAt(index);
|
|
174
|
+
if (unit < 55296 || unit > 57343) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const next = value.charCodeAt(index + 1);
|
|
178
|
+
if (unit <= 56319 && next >= 56320 && next <= 57343) {
|
|
179
|
+
index++;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
return index;
|
|
183
|
+
}
|
|
184
|
+
return -1;
|
|
185
|
+
}
|
|
186
|
+
function checkNumber(value, subject, context) {
|
|
187
|
+
if (numberProblem(value) === null) {
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (subject === null && typeof value === "number") {
|
|
191
|
+
throw new QvdValidationError("NaN and Infinity cannot be stored in a QVD field", {
|
|
192
|
+
...context,
|
|
193
|
+
provided: String(value)
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
throw new QvdValidationError(`${subject ?? "A number"} must be a finite number; got ${describeType(value)}`, {
|
|
197
|
+
...context,
|
|
198
|
+
type: typeof value
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
function checkText(value, subject, context) {
|
|
202
|
+
const problem = textProblem(value);
|
|
203
|
+
if (problem === null) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (problem.reason === "type") {
|
|
207
|
+
throw new QvdValidationError(`${subject} must be a string; got ${describeType(value)}`, {
|
|
208
|
+
...context,
|
|
209
|
+
type: typeof value
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (problem.reason === "surrogate") {
|
|
213
|
+
throw new QvdValidationError(`${subject} cannot contain an unpaired surrogate`, {
|
|
214
|
+
...context,
|
|
215
|
+
position: problem.position
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
throw new QvdValidationError(`${subject} cannot contain a NUL character`, { ...context, position: problem.position });
|
|
219
|
+
}
|
|
220
|
+
function constructorName(value) {
|
|
221
|
+
try {
|
|
222
|
+
const name = value.constructor?.name;
|
|
223
|
+
return typeof name === "string" && name !== "" ? name : null;
|
|
224
|
+
} catch {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function describeType(value) {
|
|
229
|
+
if (value === null) {
|
|
230
|
+
return "null";
|
|
231
|
+
}
|
|
232
|
+
if (typeof value === "number") {
|
|
233
|
+
return Number.isFinite(value) ? "a number" : String(value);
|
|
234
|
+
}
|
|
235
|
+
if (typeof value === "undefined") {
|
|
236
|
+
return "undefined";
|
|
237
|
+
}
|
|
238
|
+
if (typeof value !== "object") {
|
|
239
|
+
return `a ${typeof value}`;
|
|
240
|
+
}
|
|
241
|
+
let name;
|
|
242
|
+
try {
|
|
243
|
+
if (Array.isArray(value)) {
|
|
244
|
+
return "an array";
|
|
245
|
+
}
|
|
246
|
+
name = constructorName(value) ?? Object.prototype.toString.call(value).slice(8, -1);
|
|
247
|
+
if (name === "Object") {
|
|
248
|
+
if (!isPlainObject(value)) {
|
|
249
|
+
return "an object whose prototype is not Object.prototype";
|
|
114
250
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
* @return {number|null} The integer value.
|
|
119
|
-
*/
|
|
120
|
-
get intValue() {
|
|
121
|
-
return this._intValue;
|
|
251
|
+
const keys = Object.keys(value);
|
|
252
|
+
if (keys.length === 0) {
|
|
253
|
+
return "a plain object";
|
|
122
254
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
255
|
+
return `a plain object with keys ${keys.slice(0, 5).join(", ")}${keys.length > 5 ? ", ..." : ""}`;
|
|
256
|
+
}
|
|
257
|
+
} catch {
|
|
258
|
+
return "an object";
|
|
259
|
+
}
|
|
260
|
+
return `${/^[aeio]/i.test(name) ? "an" : "a"} ${name}`;
|
|
261
|
+
}
|
|
262
|
+
var INT32_MIN, INT32_MAX, NUL, DUAL_BRAND;
|
|
263
|
+
var init_cellRules = __esm({
|
|
264
|
+
"src/util/cellRules.js"() {
|
|
265
|
+
init_QvdErrors();
|
|
266
|
+
INT32_MIN = -2147483648;
|
|
267
|
+
INT32_MAX = 2147483647;
|
|
268
|
+
NUL = String.fromCharCode(0);
|
|
269
|
+
DUAL_BRAND = /* @__PURE__ */ Symbol.for("qvdjs.QvdDual");
|
|
270
|
+
__name(asDual, "asDual");
|
|
271
|
+
__name(isPlainObject, "isPlainObject");
|
|
272
|
+
__name(isNumericText, "isNumericText");
|
|
273
|
+
__name(isStoredAsInt, "isStoredAsInt");
|
|
274
|
+
__name(numberProblem, "numberProblem");
|
|
275
|
+
__name(textProblem, "textProblem");
|
|
276
|
+
__name(unpairedSurrogateIndex, "unpairedSurrogateIndex");
|
|
277
|
+
__name(checkNumber, "checkNumber");
|
|
278
|
+
__name(checkText, "checkText");
|
|
279
|
+
__name(constructorName, "constructorName");
|
|
280
|
+
__name(describeType, "describeType");
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
// src/util/symbolBytes.js
|
|
285
|
+
function kindOf(number, text) {
|
|
286
|
+
if (number === null) {
|
|
287
|
+
return 4;
|
|
288
|
+
}
|
|
289
|
+
if (isStoredAsInt(number)) {
|
|
290
|
+
return text === null ? 1 : 5;
|
|
291
|
+
}
|
|
292
|
+
return text === null ? 2 : 6;
|
|
293
|
+
}
|
|
294
|
+
function symbolByteLength(kind, number, text) {
|
|
295
|
+
const numberBytes = kind === 1 || kind === 5 ? 4 : kind === 2 || kind === 6 ? 8 : 0;
|
|
296
|
+
const textBytes = kind >= 4 ? Buffer.byteLength(text, "utf8") + 1 : 0;
|
|
297
|
+
return 1 + numberBytes + textBytes;
|
|
298
|
+
}
|
|
299
|
+
function writeSymbol(buffer, offset, kind, number, text) {
|
|
300
|
+
buffer[offset++] = kind;
|
|
301
|
+
if (kind === 1 || kind === 5) {
|
|
302
|
+
offset = buffer.writeInt32LE(number, offset);
|
|
303
|
+
} else if (kind === 2 || kind === 6) {
|
|
304
|
+
offset = buffer.writeDoubleLE(number, offset);
|
|
305
|
+
}
|
|
306
|
+
if (kind >= 4) {
|
|
307
|
+
offset += buffer.write(text, offset, "utf8");
|
|
308
|
+
buffer[offset++] = 0;
|
|
309
|
+
}
|
|
310
|
+
return offset;
|
|
311
|
+
}
|
|
312
|
+
var init_symbolBytes = __esm({
|
|
313
|
+
"src/util/symbolBytes.js"() {
|
|
314
|
+
init_cellRules();
|
|
315
|
+
__name(kindOf, "kindOf");
|
|
316
|
+
__name(symbolByteLength, "symbolByteLength");
|
|
317
|
+
__name(writeSymbol, "writeSymbol");
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// src/QvdDual.js
|
|
322
|
+
function defineHalves(target, number, text) {
|
|
323
|
+
Object.defineProperties(target, {
|
|
324
|
+
number: { value: number, enumerable: true },
|
|
325
|
+
text: { value: text, enumerable: true }
|
|
326
|
+
});
|
|
327
|
+
Object.freeze(target);
|
|
328
|
+
}
|
|
329
|
+
function dualFromSymbol(number, text) {
|
|
330
|
+
const dual = Object.create(QvdDual.prototype);
|
|
331
|
+
defineHalves(dual, number, text);
|
|
332
|
+
return dual;
|
|
333
|
+
}
|
|
334
|
+
var QvdDual;
|
|
335
|
+
var init_QvdDual = __esm({
|
|
336
|
+
"src/QvdDual.js"() {
|
|
337
|
+
init_cellRules();
|
|
338
|
+
QvdDual = class {
|
|
339
|
+
static {
|
|
340
|
+
__name(this, "QvdDual");
|
|
130
341
|
}
|
|
131
342
|
/**
|
|
132
|
-
*
|
|
343
|
+
* Constructs a dual value.
|
|
133
344
|
*
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
get stringValue() {
|
|
137
|
-
return this._stringValue;
|
|
138
|
-
}
|
|
139
|
-
/**
|
|
140
|
-
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
141
|
-
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
345
|
+
* The storage kind is not chosen here. The writer derives it from the number, as Qlik does: an
|
|
346
|
+
* integer inside the int32 range is stored as a dual int, anything else as a dual double.
|
|
142
347
|
*
|
|
143
|
-
* @
|
|
348
|
+
* @param {number} number The numeric half. Must be a finite number.
|
|
349
|
+
* @param {string} text The text half. Must be a string with no NUL and no unpaired surrogate.
|
|
350
|
+
* @throws {QvdValidationError} If either half cannot be stored in a QVD. The message names the half.
|
|
144
351
|
*/
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
return this._intValue;
|
|
150
|
-
} else if (null != this._doubleValue) {
|
|
151
|
-
return this._doubleValue;
|
|
152
|
-
} else {
|
|
153
|
-
return null;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
/**
|
|
157
|
-
* Converts the symbol to its byte representation.
|
|
158
|
-
*
|
|
159
|
-
* @return {Buffer} The byte representation of the symbol.
|
|
160
|
-
*/
|
|
161
|
-
toByteRepresentation() {
|
|
162
|
-
if (this._intValue !== null && this._stringValue !== null) {
|
|
163
|
-
const intBuffer = Buffer.alloc(4);
|
|
164
|
-
intBuffer.writeInt32LE(this._intValue);
|
|
165
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
166
|
-
return Buffer.concat([Buffer.from([5]), intBuffer, stringBuffer]);
|
|
167
|
-
} else if (this._doubleValue !== null && this._stringValue !== null) {
|
|
168
|
-
const floatBuffer = Buffer.alloc(8);
|
|
169
|
-
floatBuffer.writeDoubleLE(this._doubleValue);
|
|
170
|
-
const stringBuffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
171
|
-
return Buffer.concat([Buffer.from([6]), floatBuffer, stringBuffer]);
|
|
172
|
-
} else if (this._intValue !== null) {
|
|
173
|
-
const buffer = Buffer.alloc(4);
|
|
174
|
-
buffer.writeInt32LE(this._intValue);
|
|
175
|
-
return Buffer.concat([Buffer.from([1]), buffer]);
|
|
176
|
-
} else if (this._doubleValue !== null) {
|
|
177
|
-
const buffer = Buffer.alloc(8);
|
|
178
|
-
buffer.writeDoubleLE(this._doubleValue);
|
|
179
|
-
return Buffer.concat([Buffer.from([2]), buffer]);
|
|
180
|
-
} else if (this._stringValue !== null) {
|
|
181
|
-
const buffer = Buffer.concat([Buffer.from(this._stringValue, "utf-8"), Buffer.from([0])]);
|
|
182
|
-
return Buffer.concat([Buffer.from([4]), buffer]);
|
|
183
|
-
} else {
|
|
184
|
-
throw new QvdValidationError("The symbol does not contain any value.", {
|
|
185
|
-
intValue: this._intValue,
|
|
186
|
-
doubleValue: this._doubleValue,
|
|
187
|
-
stringValue: this._stringValue
|
|
188
|
-
});
|
|
189
|
-
}
|
|
352
|
+
constructor(number, text) {
|
|
353
|
+
checkNumber(number, "The number of a dual value", { half: "number" });
|
|
354
|
+
checkText(text, "The text of a dual value", { half: "text" });
|
|
355
|
+
defineHalves(this, number, text);
|
|
190
356
|
}
|
|
191
357
|
/**
|
|
192
|
-
*
|
|
358
|
+
* Refuses every implicit conversion. See the class comment for why neither half is a safe answer.
|
|
193
359
|
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*/
|
|
197
|
-
equals(value) {
|
|
198
|
-
if (!(value instanceof _QvdSymbol)) {
|
|
199
|
-
return false;
|
|
200
|
-
}
|
|
201
|
-
return this._intValue === value.intValue && this._doubleValue === value.doubleValue && this._stringValue === value.stringValue;
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Constructs a pure integer value symbol.
|
|
360
|
+
* The hint JavaScript passes is not used: `'number'`, `'string'` and `'default'` say which
|
|
361
|
+
* conversion ran, not which half the caller meant, and the fix is the same for all three.
|
|
205
362
|
*
|
|
206
|
-
* @
|
|
207
|
-
* @
|
|
363
|
+
* @return {never}
|
|
364
|
+
* @throws {TypeError} Always.
|
|
208
365
|
*/
|
|
209
|
-
|
|
210
|
-
|
|
366
|
+
[Symbol.toPrimitive]() {
|
|
367
|
+
throw new TypeError(
|
|
368
|
+
`A QvdDual holds two values, ${this.number} and ${JSON.stringify(this.text)}; use .number or .text`
|
|
369
|
+
);
|
|
211
370
|
}
|
|
212
371
|
/**
|
|
213
|
-
*
|
|
372
|
+
* Both halves, so `JSON.stringify` loses neither - and produces the shape the writer accepts back.
|
|
214
373
|
*
|
|
215
|
-
* @
|
|
216
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
374
|
+
* @return {{number: number, text: string}} The dual as a plain object.
|
|
217
375
|
*/
|
|
218
|
-
|
|
219
|
-
return
|
|
376
|
+
toJSON() {
|
|
377
|
+
return { number: this.number, text: this.text };
|
|
220
378
|
}
|
|
221
379
|
/**
|
|
222
|
-
*
|
|
380
|
+
* How Node's `util.inspect` and `console.log` show a dual.
|
|
223
381
|
*
|
|
224
|
-
* @
|
|
225
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
382
|
+
* @return {string} For example `QvdDual(4.5, "4.50")`.
|
|
226
383
|
*/
|
|
227
|
-
|
|
228
|
-
return
|
|
384
|
+
[/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
|
|
385
|
+
return `QvdDual(${this.number}, ${JSON.stringify(this.text)})`;
|
|
229
386
|
}
|
|
230
|
-
/**
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
* @param {number} intValue The integer value.
|
|
234
|
-
* @param {string} stringValue The string value.
|
|
235
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
236
|
-
*/
|
|
237
|
-
static fromDualIntValue(intValue, stringValue) {
|
|
238
|
-
return new _QvdSymbol(intValue, null, stringValue);
|
|
387
|
+
/** @return {string} `'QvdDual'`, for `Object.prototype.toString`. */
|
|
388
|
+
get [Symbol.toStringTag]() {
|
|
389
|
+
return "QvdDual";
|
|
239
390
|
}
|
|
240
391
|
/**
|
|
241
|
-
*
|
|
392
|
+
* Whether a value is a dual cell: a `QvdDual` from any copy of this library, or a plain object whose
|
|
393
|
+
* own enumerable keys are exactly `number` and `text`, which is what a clone of one becomes.
|
|
394
|
+
*
|
|
395
|
+
* Recognition only. The halves are checked when the value is written.
|
|
242
396
|
*
|
|
243
|
-
* @param {
|
|
244
|
-
* @
|
|
245
|
-
* @return {QvdSymbol} The constructed value symbol.
|
|
397
|
+
* @param {any} value Any value.
|
|
398
|
+
* @return {boolean} True for a dual cell.
|
|
246
399
|
*/
|
|
247
|
-
static
|
|
248
|
-
return
|
|
400
|
+
static isDual(value) {
|
|
401
|
+
return asDual(value) !== null;
|
|
249
402
|
}
|
|
250
403
|
};
|
|
404
|
+
Object.defineProperty(QvdDual.prototype, DUAL_BRAND, { value: true });
|
|
405
|
+
__name(defineHalves, "defineHalves");
|
|
406
|
+
__name(dualFromSymbol, "dualFromSymbol");
|
|
251
407
|
}
|
|
252
408
|
});
|
|
253
409
|
|
|
@@ -347,12 +503,41 @@ function selectFields(fields, requested, filePath) {
|
|
|
347
503
|
return fields[index];
|
|
348
504
|
});
|
|
349
505
|
}
|
|
506
|
+
function normaliseDuals(value, filePath) {
|
|
507
|
+
if (value === void 0 || value === null) {
|
|
508
|
+
return "number";
|
|
509
|
+
}
|
|
510
|
+
if (!DUAL_MODES.includes(value)) {
|
|
511
|
+
throw new QvdValidationError(`duals must be one of ${DUAL_MODES.map((mode) => `'${mode}'`).join(", ")}`, {
|
|
512
|
+
option: "duals",
|
|
513
|
+
provided: value,
|
|
514
|
+
file: filePath
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
return value;
|
|
518
|
+
}
|
|
519
|
+
function normaliseCoerceNumericStrings(value, filePath) {
|
|
520
|
+
if (value === void 0 || value === null) {
|
|
521
|
+
return false;
|
|
522
|
+
}
|
|
523
|
+
if (typeof value !== "boolean") {
|
|
524
|
+
throw new QvdValidationError("coerceNumericStrings must be true or false", {
|
|
525
|
+
option: "coerceNumericStrings",
|
|
526
|
+
provided: value,
|
|
527
|
+
type: typeof value,
|
|
528
|
+
file: filePath
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
return value;
|
|
532
|
+
}
|
|
350
533
|
function readerOptionsFrom(options) {
|
|
351
534
|
return {
|
|
352
535
|
allowedDir: options.allowedDir,
|
|
353
536
|
memorySafetyFactor: options.memorySafetyFactor,
|
|
354
537
|
symbolFilteringThreshold: options.symbolFilteringThreshold,
|
|
355
538
|
fields: options.fields === void 0 ? null : options.fields,
|
|
539
|
+
duals: options.duals,
|
|
540
|
+
coerceNumericStrings: options.coerceNumericStrings,
|
|
356
541
|
onProgress: options.onProgress,
|
|
357
542
|
signal: options.signal
|
|
358
543
|
};
|
|
@@ -367,9 +552,151 @@ function metadataOptionsFrom(options) {
|
|
|
367
552
|
function windowFrom(options) {
|
|
368
553
|
return { offset: options.offset, limit: options.limit, maxRows: options.maxRows };
|
|
369
554
|
}
|
|
555
|
+
var DUAL_MODES;
|
|
370
556
|
var init_readOptions = __esm({
|
|
371
557
|
"src/util/readOptions.js"() {
|
|
372
558
|
init_QvdErrors();
|
|
559
|
+
__name(requireRowCount, "requireRowCount");
|
|
560
|
+
__name(normaliseWindow, "normaliseWindow");
|
|
561
|
+
__name(resolveWindow, "resolveWindow");
|
|
562
|
+
__name(selectFields, "selectFields");
|
|
563
|
+
DUAL_MODES = Object.freeze(["number", "text", "both"]);
|
|
564
|
+
__name(normaliseDuals, "normaliseDuals");
|
|
565
|
+
__name(normaliseCoerceNumericStrings, "normaliseCoerceNumericStrings");
|
|
566
|
+
__name(readerOptionsFrom, "readerOptionsFrom");
|
|
567
|
+
__name(metadataOptionsFrom, "metadataOptionsFrom");
|
|
568
|
+
__name(windowFrom, "windowFrom");
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
// src/util/storedSymbols.js
|
|
573
|
+
function trustStoredSymbols(entries) {
|
|
574
|
+
const record = Object.freeze(entries);
|
|
575
|
+
trusted.add(record);
|
|
576
|
+
return record;
|
|
577
|
+
}
|
|
578
|
+
function attachStoredSymbols(metadata, record) {
|
|
579
|
+
if (metadata !== null && typeof metadata === "object" && Object.isExtensible(metadata)) {
|
|
580
|
+
Object.defineProperty(metadata, STORED_SYMBOLS, { value: record, enumerable: false, configurable: true });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
function refuse(message, context) {
|
|
584
|
+
throw new QvdValidationError(message, context);
|
|
585
|
+
}
|
|
586
|
+
function normaliseStoredSymbols(record) {
|
|
587
|
+
if (record === null || record === void 0) {
|
|
588
|
+
return null;
|
|
589
|
+
}
|
|
590
|
+
if (typeof record === "object" && trusted.has(record)) {
|
|
591
|
+
return record;
|
|
592
|
+
}
|
|
593
|
+
if (!Array.isArray(record)) {
|
|
594
|
+
refuse(`storedSymbols must be an array of field entries; got ${describeType(record)}`, {
|
|
595
|
+
option: "storedSymbols",
|
|
596
|
+
type: typeof record
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
const fields = /* @__PURE__ */ new Set();
|
|
600
|
+
const entries = record.map((entry, index) => {
|
|
601
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry) || typeof entry.field !== "string") {
|
|
602
|
+
refuse("Each storedSymbols entry must be an object with a string field name", { entry: index });
|
|
603
|
+
}
|
|
604
|
+
const { field, values, numbers, texts } = entry;
|
|
605
|
+
if (fields.has(field)) {
|
|
606
|
+
refuse(`storedSymbols lists field '${field}' twice`, { field, entry: index });
|
|
607
|
+
}
|
|
608
|
+
fields.add(field);
|
|
609
|
+
if (!Array.isArray(values) || !Array.isArray(numbers) || !Array.isArray(texts) || values.length !== numbers.length || values.length !== texts.length) {
|
|
610
|
+
refuse(`The values, numbers and texts of the storedSymbols entry for '${field}' must be arrays of one length`, {
|
|
611
|
+
field,
|
|
612
|
+
entry: index
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
for (let symbol = 0; symbol < values.length; symbol++) {
|
|
616
|
+
const value = values[symbol];
|
|
617
|
+
const number = numbers[symbol];
|
|
618
|
+
const text = texts[symbol];
|
|
619
|
+
const context = { field, symbol };
|
|
620
|
+
if (typeof value !== "number" && typeof value !== "string") {
|
|
621
|
+
refuse(`A stored symbol's value must be a number or a string; got ${describeType(value)}`, {
|
|
622
|
+
...context,
|
|
623
|
+
type: typeof value
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
if (number !== null) {
|
|
627
|
+
checkNumber(number, "A stored symbol's number", context);
|
|
628
|
+
}
|
|
629
|
+
if (text !== null) {
|
|
630
|
+
checkText(text, "A stored symbol's text", context);
|
|
631
|
+
}
|
|
632
|
+
const consistent = number !== null && text !== null ? sameValueZero(value, number) || value === text : number === null && text !== null ? value === text || typeof value === "number" && Number.isFinite(value) : number !== null && sameValueZero(value, number);
|
|
633
|
+
if (!consistent) {
|
|
634
|
+
refuse(
|
|
635
|
+
number === null && text === null ? "A stored symbol needs a number or a text" : "A stored symbol's value must be its number or its text",
|
|
636
|
+
{ ...context, value, number, text }
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
return Object.freeze({
|
|
641
|
+
field,
|
|
642
|
+
values: Object.freeze(values.slice()),
|
|
643
|
+
numbers: Object.freeze(numbers.slice()),
|
|
644
|
+
texts: Object.freeze(texts.slice())
|
|
645
|
+
});
|
|
646
|
+
});
|
|
647
|
+
return trustStoredSymbols(entries);
|
|
648
|
+
}
|
|
649
|
+
function narrowStoredSymbols(record, columns) {
|
|
650
|
+
if (record === null) {
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
const kept = record.filter((entry) => columns.includes(entry.field));
|
|
654
|
+
return kept.length === record.length ? record : trustStoredSymbols(kept);
|
|
655
|
+
}
|
|
656
|
+
function storedSymbolsEntry(record, field) {
|
|
657
|
+
if (record === null) {
|
|
658
|
+
return null;
|
|
659
|
+
}
|
|
660
|
+
return record.find((entry) => entry.field === field) ?? null;
|
|
661
|
+
}
|
|
662
|
+
function firstTextByValue(entry) {
|
|
663
|
+
const byValue = /* @__PURE__ */ new Map();
|
|
664
|
+
for (let index = entry.values.length - 1; index >= 0; index--) {
|
|
665
|
+
const text = entry.texts[index];
|
|
666
|
+
if (text !== null) {
|
|
667
|
+
byValue.set(entry.values[index], text);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
return byValue;
|
|
671
|
+
}
|
|
672
|
+
function storedTextOf(entry, value) {
|
|
673
|
+
let byValue = firstTexts.get(entry);
|
|
674
|
+
if (byValue === void 0) {
|
|
675
|
+
byValue = firstTextByValue(entry);
|
|
676
|
+
firstTexts.set(entry, byValue);
|
|
677
|
+
}
|
|
678
|
+
return byValue.get(value) ?? null;
|
|
679
|
+
}
|
|
680
|
+
function sameValueZero(a, b) {
|
|
681
|
+
return a === b || a !== a && b !== b;
|
|
682
|
+
}
|
|
683
|
+
var STORED_SYMBOLS, trusted, firstTexts;
|
|
684
|
+
var init_storedSymbols = __esm({
|
|
685
|
+
"src/util/storedSymbols.js"() {
|
|
686
|
+
init_QvdErrors();
|
|
687
|
+
init_cellRules();
|
|
688
|
+
STORED_SYMBOLS = /* @__PURE__ */ Symbol.for("qvdjs.storedSymbols");
|
|
689
|
+
trusted = /* @__PURE__ */ new WeakSet();
|
|
690
|
+
__name(trustStoredSymbols, "trustStoredSymbols");
|
|
691
|
+
__name(attachStoredSymbols, "attachStoredSymbols");
|
|
692
|
+
__name(refuse, "refuse");
|
|
693
|
+
__name(normaliseStoredSymbols, "normaliseStoredSymbols");
|
|
694
|
+
__name(narrowStoredSymbols, "narrowStoredSymbols");
|
|
695
|
+
__name(storedSymbolsEntry, "storedSymbolsEntry");
|
|
696
|
+
firstTexts = /* @__PURE__ */ new WeakMap();
|
|
697
|
+
__name(firstTextByValue, "firstTextByValue");
|
|
698
|
+
__name(storedTextOf, "storedTextOf");
|
|
699
|
+
__name(sameValueZero, "sameValueZero");
|
|
373
700
|
}
|
|
374
701
|
});
|
|
375
702
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
@@ -409,7 +736,7 @@ function resolveDeepestExisting(target) {
|
|
|
409
736
|
function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
410
737
|
let baseStat;
|
|
411
738
|
try {
|
|
412
|
-
baseStat = fs.statSync(fs.realpathSync(resolvedBaseDir));
|
|
739
|
+
baseStat = fs.statSync(fs.realpathSync(resolvedBaseDir), { bigint: true });
|
|
413
740
|
} catch {
|
|
414
741
|
return null;
|
|
415
742
|
}
|
|
@@ -420,7 +747,7 @@ function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
|
420
747
|
for (; ; ) {
|
|
421
748
|
let stat;
|
|
422
749
|
try {
|
|
423
|
-
stat = fs.statSync(current);
|
|
750
|
+
stat = fs.statSync(current, { bigint: true });
|
|
424
751
|
} catch {
|
|
425
752
|
return null;
|
|
426
753
|
}
|
|
@@ -474,6 +801,10 @@ function validatePath(filePath, allowedDir) {
|
|
|
474
801
|
var init_validatePath = __esm({
|
|
475
802
|
"src/util/validatePath.js"() {
|
|
476
803
|
init_QvdErrors();
|
|
804
|
+
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
805
|
+
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
806
|
+
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
807
|
+
__name(validatePath, "validatePath");
|
|
477
808
|
}
|
|
478
809
|
});
|
|
479
810
|
|
|
@@ -523,6 +854,9 @@ var init_bitUtils = __esm({
|
|
|
523
854
|
"src/util/bitUtils.js"() {
|
|
524
855
|
MAX_BIT_WIDTH = 31;
|
|
525
856
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
857
|
+
__name(fieldGeometry, "fieldGeometry");
|
|
858
|
+
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
859
|
+
__name(writeBitField, "writeBitField");
|
|
526
860
|
}
|
|
527
861
|
});
|
|
528
862
|
|
|
@@ -531,14 +865,310 @@ var QvdFileWriter_exports = {};
|
|
|
531
865
|
__export(QvdFileWriter_exports, {
|
|
532
866
|
QvdFileWriter: () => QvdFileWriter
|
|
533
867
|
});
|
|
534
|
-
|
|
868
|
+
function notXmlIndex(value) {
|
|
869
|
+
for (let index = 0; index < value.length; index++) {
|
|
870
|
+
const unit = value.charCodeAt(index);
|
|
871
|
+
if (unit < 32 && unit !== 9 && unit !== 10 && unit !== 13 || unit === 65534 || unit === 65535) {
|
|
872
|
+
return index;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
return -1;
|
|
876
|
+
}
|
|
877
|
+
function checkHeaderText(value, subject, context) {
|
|
878
|
+
checkText(value, subject, context);
|
|
879
|
+
const position = notXmlIndex(value);
|
|
880
|
+
if (position !== -1) {
|
|
881
|
+
const code = value.charCodeAt(position).toString(16).toUpperCase().padStart(4, "0");
|
|
882
|
+
throw new QvdValidationError(`${subject} cannot contain U+${code}, which XML cannot hold`, { ...context, position });
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
function checkHeaderTexts(value, property, owner, context) {
|
|
886
|
+
if (typeof value === "string") {
|
|
887
|
+
checkHeaderText(value, `The ${property} of ${owner}`, { ...context, property });
|
|
888
|
+
} else if (Array.isArray(value)) {
|
|
889
|
+
value.forEach((item, index) => checkHeaderTexts(item, `${property}[${index}]`, owner, context));
|
|
890
|
+
} else if (value !== null && typeof value === "object") {
|
|
891
|
+
for (const [key, item] of Object.entries(value)) {
|
|
892
|
+
checkHeaderTexts(item, `${property}.${key}`, owner, context);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
function validateColumnNames(columns, filePath) {
|
|
897
|
+
const seen = /* @__PURE__ */ new Set();
|
|
898
|
+
columns.forEach((name, index) => {
|
|
899
|
+
if (typeof name !== "string" || name.length === 0) {
|
|
900
|
+
throw new QvdValidationError("Field names must be non-empty strings", {
|
|
901
|
+
column: index,
|
|
902
|
+
provided: name,
|
|
903
|
+
type: typeof name,
|
|
904
|
+
file: filePath,
|
|
905
|
+
stage: "buildSymbolTable"
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
checkHeaderText(name, "A field name", { column: index, provided: name, file: filePath, stage: "buildSymbolTable" });
|
|
909
|
+
if (seen.has(name)) {
|
|
910
|
+
throw new QvdValidationError(`Field '${name}' appears twice`, {
|
|
911
|
+
column: name,
|
|
912
|
+
columnIndex: index,
|
|
913
|
+
file: filePath,
|
|
914
|
+
stage: "buildSymbolTable"
|
|
915
|
+
});
|
|
916
|
+
}
|
|
917
|
+
seen.add(name);
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
function refuseCell(value, column, row, filePath) {
|
|
921
|
+
let resemblesDual = false;
|
|
922
|
+
try {
|
|
923
|
+
resemblesDual = value !== null && typeof value === "object" && ("number" in value || "text" in value || "intValue" in value && "stringValue" in value);
|
|
924
|
+
} catch {
|
|
925
|
+
}
|
|
926
|
+
throw new QvdValidationError(
|
|
927
|
+
`A QVD field holds numbers, strings, dual values and NULL; ${describeType(value)} cannot be written. Convert it first - a Date to new QvdDual(dateToQlikSerial(date), text), the serial and the text Qlik shows, which is how Qlik stores a date; a boolean to -1 and 0, as a Qlik comparison stores it.` + (resemblesDual ? " A dual value is a QvdDual, or an object whose only keys are number and text." : ""),
|
|
928
|
+
{
|
|
929
|
+
column,
|
|
930
|
+
row,
|
|
931
|
+
type: typeof value,
|
|
932
|
+
constructor: constructorName(value) ?? void 0,
|
|
933
|
+
file: filePath,
|
|
934
|
+
stage: "buildSymbolTable"
|
|
935
|
+
}
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
function valuesAreNumbers(entry) {
|
|
939
|
+
const { values, numbers } = entry;
|
|
940
|
+
for (let index = 0; index < values.length; index++) {
|
|
941
|
+
if (!sameValueZero(values[index], numbers[index])) {
|
|
942
|
+
return false;
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
return true;
|
|
946
|
+
}
|
|
947
|
+
function newSlot(column, key, text) {
|
|
948
|
+
const slot = column.keys.length;
|
|
949
|
+
column.keys.push(key);
|
|
950
|
+
column.texts.push(text);
|
|
951
|
+
if (typeof key === "number") {
|
|
952
|
+
column.facts.hasNumber = true;
|
|
953
|
+
if (!Number.isInteger(key)) column.facts.hasFraction = true;
|
|
954
|
+
} else {
|
|
955
|
+
column.facts.hasNonNumber = true;
|
|
956
|
+
}
|
|
957
|
+
return slot;
|
|
958
|
+
}
|
|
959
|
+
function slotFor(column, key, text) {
|
|
960
|
+
let slot = column.byKey.get(key);
|
|
961
|
+
if (slot === void 0) {
|
|
962
|
+
slot = newSlot(column, key, text);
|
|
963
|
+
column.byKey.set(key, slot);
|
|
964
|
+
} else if (text !== null && column.texts[slot] === null && typeof key === "number") {
|
|
965
|
+
column.texts[slot] = text;
|
|
966
|
+
}
|
|
967
|
+
return slot;
|
|
968
|
+
}
|
|
969
|
+
function standsForSeveral(indices, numbers, texts) {
|
|
970
|
+
let number = null;
|
|
971
|
+
let string = null;
|
|
972
|
+
for (const index of indices) {
|
|
973
|
+
if (numbers[index] !== null) {
|
|
974
|
+
if (number === null) {
|
|
975
|
+
number = numbers[index];
|
|
976
|
+
} else if (!sameValueZero(number, numbers[index])) {
|
|
977
|
+
return true;
|
|
978
|
+
}
|
|
979
|
+
} else if (string === null) {
|
|
980
|
+
string = texts[index];
|
|
981
|
+
} else if (string !== texts[index]) {
|
|
982
|
+
return true;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
return number !== null && string !== null;
|
|
986
|
+
}
|
|
987
|
+
function ambiguousAsUncoercedText(entry) {
|
|
988
|
+
const { values, numbers, texts } = entry;
|
|
989
|
+
for (let index = 0; index < values.length; index++) {
|
|
990
|
+
if (typeof values[index] === "number" && numbers[index] !== null && texts[index] !== null) {
|
|
991
|
+
if (!isNumericText(texts[index])) {
|
|
992
|
+
return false;
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
const byShown = /* @__PURE__ */ new Map();
|
|
997
|
+
for (let index = 0; index < values.length; index++) {
|
|
998
|
+
const shown = texts[index] ?? numbers[index];
|
|
999
|
+
const indices = byShown.get(shown);
|
|
1000
|
+
if (indices === void 0) {
|
|
1001
|
+
byShown.set(shown, [index]);
|
|
1002
|
+
} else {
|
|
1003
|
+
indices.push(index);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
for (const indices of byShown.values()) {
|
|
1007
|
+
if (indices.length > 1 && standsForSeveral(indices, numbers, texts)) {
|
|
1008
|
+
return true;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
return false;
|
|
1012
|
+
}
|
|
1013
|
+
function unambiguousWithoutDuals(column) {
|
|
1014
|
+
const { numbers, texts } = column.entry;
|
|
1015
|
+
for (const found of column.byValue.values()) {
|
|
1016
|
+
if (typeof found !== "number") {
|
|
1017
|
+
const kept = found.filter((index) => numbers[index] === null || texts[index] === null);
|
|
1018
|
+
if (kept.length > 1 && standsForSeveral(kept, numbers, texts)) {
|
|
1019
|
+
return false;
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
return true;
|
|
1024
|
+
}
|
|
1025
|
+
function ambiguityRemedy(column) {
|
|
1026
|
+
const { values, numbers, texts } = column.entry;
|
|
1027
|
+
let coerced = false;
|
|
1028
|
+
let other = false;
|
|
1029
|
+
for (const [value, found] of column.byValue) {
|
|
1030
|
+
if (typeof found !== "number" && standsForSeveral(found, numbers, texts)) {
|
|
1031
|
+
if (typeof value === "number") {
|
|
1032
|
+
coerced = true;
|
|
1033
|
+
} else {
|
|
1034
|
+
other = true;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
if (!coerced) {
|
|
1039
|
+
return "Read the field with {duals: 'both'}, or write QvdDual cells.";
|
|
1040
|
+
}
|
|
1041
|
+
if (!other && !ambiguousAsUncoercedText(column.entry)) {
|
|
1042
|
+
return "Read the field without {coerceNumericStrings: true}.";
|
|
1043
|
+
}
|
|
1044
|
+
if (unambiguousWithoutDuals(column)) {
|
|
1045
|
+
return "Read the field with {duals: 'both'}.";
|
|
1046
|
+
}
|
|
1047
|
+
return values.some((value, index) => typeof value === "string" && numbers[index] !== null) ? "Read the field with {duals: 'both'} and without {coerceNumericStrings: true}." : "Read the field without {coerceNumericStrings: true}, and with {duals: 'both'} as well if it was read with {duals: 'text'}.";
|
|
1048
|
+
}
|
|
1049
|
+
function slotForCell(column, value, row, filePath) {
|
|
1050
|
+
if (column.byCell === column.byKey) {
|
|
1051
|
+
return newSlot(column, value, column.textByNumber === null ? null : column.textByNumber.get(value) ?? null);
|
|
1052
|
+
}
|
|
1053
|
+
const found = column.byValue.get(value);
|
|
1054
|
+
if (found === void 0) {
|
|
1055
|
+
return slotFor(column, value, null);
|
|
1056
|
+
}
|
|
1057
|
+
const { numbers, texts } = column.entry;
|
|
1058
|
+
const indices = typeof found === "number" ? [found] : found;
|
|
1059
|
+
if (standsForSeveral(indices, numbers, texts)) {
|
|
1060
|
+
const stored = [];
|
|
1061
|
+
for (const index of indices) {
|
|
1062
|
+
if (!stored.some((pair) => sameValueZero(pair.number, numbers[index]) && pair.text === texts[index])) {
|
|
1063
|
+
stored.push({ number: numbers[index], text: texts[index] });
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
throw new QvdValidationError(
|
|
1067
|
+
`The value ${JSON.stringify(value)} in field '${column.name}' (row ${row}) was read from ${stored.length} different stored values, so writing it back would have to guess which one. ${ambiguityRemedy(column)}`,
|
|
1068
|
+
{ column: column.name, row, value, stored, file: filePath, stage: "buildSymbolTable" }
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
let number = null;
|
|
1072
|
+
let numberText = null;
|
|
1073
|
+
for (const index of indices) {
|
|
1074
|
+
if (numbers[index] === null) {
|
|
1075
|
+
return slotFor(column, texts[index], null);
|
|
1076
|
+
}
|
|
1077
|
+
number ??= numbers[index];
|
|
1078
|
+
numberText ??= texts[index];
|
|
1079
|
+
}
|
|
1080
|
+
return slotFor(column, number, numberText);
|
|
1081
|
+
}
|
|
1082
|
+
function slotForObject(column, value, row, filePath) {
|
|
1083
|
+
const dual = asDual(value);
|
|
1084
|
+
if (dual === null) {
|
|
1085
|
+
refuseCell(value, column.name, row, filePath);
|
|
1086
|
+
}
|
|
1087
|
+
const { number, text } = dual;
|
|
1088
|
+
if (numberProblem(number) !== null) {
|
|
1089
|
+
checkNumber(number, "The number of a dual value", {
|
|
1090
|
+
column: column.name,
|
|
1091
|
+
row,
|
|
1092
|
+
half: "number",
|
|
1093
|
+
file: filePath,
|
|
1094
|
+
stage: "buildSymbolTable"
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
if (textProblem(text) !== null) {
|
|
1098
|
+
checkText(text, "The text of a dual value", {
|
|
1099
|
+
column: column.name,
|
|
1100
|
+
row,
|
|
1101
|
+
half: "text",
|
|
1102
|
+
file: filePath,
|
|
1103
|
+
stage: "buildSymbolTable"
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
const slot = slotFor(column, number, text);
|
|
1107
|
+
if (column.byObject.size < OBJECT_MEMO_BASE + 2 * column.keys.length) {
|
|
1108
|
+
column.byObject.set(value, slot);
|
|
1109
|
+
}
|
|
1110
|
+
return slot;
|
|
1111
|
+
}
|
|
1112
|
+
function pruneContradictedTags(tags, facts) {
|
|
1113
|
+
if (tags === null || typeof tags !== "object" || tags.String === void 0 || facts === void 0) {
|
|
1114
|
+
return tags || {};
|
|
1115
|
+
}
|
|
1116
|
+
const list = Array.isArray(tags.String) ? tags.String : [tags.String];
|
|
1117
|
+
const kept = list.filter((tag) => {
|
|
1118
|
+
if (facts.hasNonNumber && NUMERIC_TAGS.has(tag)) return false;
|
|
1119
|
+
if (facts.hasFraction && WHOLE_NUMBER_TAGS.has(tag)) return false;
|
|
1120
|
+
if (facts.hasNumber && TEXT_TAGS.has(tag)) return false;
|
|
1121
|
+
return true;
|
|
1122
|
+
});
|
|
1123
|
+
if (kept.length === list.length) {
|
|
1124
|
+
return tags;
|
|
1125
|
+
}
|
|
1126
|
+
return kept.length === 0 ? {} : { ...tags, String: kept };
|
|
1127
|
+
}
|
|
1128
|
+
function resetContradictedNumberFormat(numberFormat, facts) {
|
|
1129
|
+
if (!numberFormat) {
|
|
1130
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1131
|
+
}
|
|
1132
|
+
if (facts !== void 0 && !facts.hasNumber && facts.hasNonNumber && NUMERIC_FORMATS.has(numberFormat.Type)) {
|
|
1133
|
+
return { ...UNKNOWN_NUMBER_FORMAT };
|
|
1134
|
+
}
|
|
1135
|
+
return numberFormat;
|
|
1136
|
+
}
|
|
1137
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, QvdFileWriter;
|
|
535
1138
|
var init_QvdFileWriter = __esm({
|
|
536
1139
|
"src/QvdFileWriter.js"() {
|
|
537
|
-
init_QvdSymbol();
|
|
538
1140
|
init_QvdErrors();
|
|
539
1141
|
init_validatePath();
|
|
540
1142
|
init_bitUtils();
|
|
541
|
-
|
|
1143
|
+
init_cellRules();
|
|
1144
|
+
init_symbolBytes();
|
|
1145
|
+
init_storedSymbols();
|
|
1146
|
+
__name(notXmlIndex, "notXmlIndex");
|
|
1147
|
+
__name(checkHeaderText, "checkHeaderText");
|
|
1148
|
+
__name(checkHeaderTexts, "checkHeaderTexts");
|
|
1149
|
+
__name(validateColumnNames, "validateColumnNames");
|
|
1150
|
+
__name(refuseCell, "refuseCell");
|
|
1151
|
+
OBJECT_MEMO_BASE = 64;
|
|
1152
|
+
__name(valuesAreNumbers, "valuesAreNumbers");
|
|
1153
|
+
__name(newSlot, "newSlot");
|
|
1154
|
+
__name(slotFor, "slotFor");
|
|
1155
|
+
__name(standsForSeveral, "standsForSeveral");
|
|
1156
|
+
__name(ambiguousAsUncoercedText, "ambiguousAsUncoercedText");
|
|
1157
|
+
__name(unambiguousWithoutDuals, "unambiguousWithoutDuals");
|
|
1158
|
+
__name(ambiguityRemedy, "ambiguityRemedy");
|
|
1159
|
+
__name(slotForCell, "slotForCell");
|
|
1160
|
+
__name(slotForObject, "slotForObject");
|
|
1161
|
+
NUMERIC_TAGS = /* @__PURE__ */ new Set(["$numeric", "$integer", "$date", "$time", "$timestamp"]);
|
|
1162
|
+
TEXT_TAGS = /* @__PURE__ */ new Set(["$text", "$ascii"]);
|
|
1163
|
+
WHOLE_NUMBER_TAGS = /* @__PURE__ */ new Set(["$integer", "$date"]);
|
|
1164
|
+
NUMERIC_FORMATS = /* @__PURE__ */ new Set(["INTEGER", "REAL", "FIX", "MONEY", "DATE", "TIME", "TIMESTAMP", "INTERVAL"]);
|
|
1165
|
+
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1166
|
+
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1167
|
+
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1168
|
+
QvdFileWriter = class {
|
|
1169
|
+
static {
|
|
1170
|
+
__name(this, "QvdFileWriter");
|
|
1171
|
+
}
|
|
542
1172
|
/**
|
|
543
1173
|
* Constructs a new QVD file writer.
|
|
544
1174
|
*
|
|
@@ -559,7 +1189,8 @@ var init_QvdFileWriter = __esm({
|
|
|
559
1189
|
this._onProgress = onProgress;
|
|
560
1190
|
this._header = null;
|
|
561
1191
|
this._symbolBuffer = null;
|
|
562
|
-
this.
|
|
1192
|
+
this._symbolCounts = null;
|
|
1193
|
+
this._symbolFacts = null;
|
|
563
1194
|
this._symbolTableMetadata = null;
|
|
564
1195
|
this._indexBuffer = null;
|
|
565
1196
|
this._symbolIndexByValue = null;
|
|
@@ -670,19 +1301,12 @@ var init_QvdFileWriter = __esm({
|
|
|
670
1301
|
BitOffset: this._indexTableMetadata?.[index][0],
|
|
671
1302
|
BitWidth: this._indexTableMetadata?.[index][1],
|
|
672
1303
|
Bias: this._indexTableMetadata?.[index][2],
|
|
673
|
-
NoOfSymbols: this.
|
|
1304
|
+
NoOfSymbols: this._symbolCounts?.[index],
|
|
674
1305
|
Offset: this._symbolTableMetadata?.[index][0],
|
|
675
1306
|
Length: this._symbolTableMetadata?.[index][1],
|
|
676
1307
|
Comment: existingField?.Comment || "",
|
|
677
|
-
NumberFormat: existingField?.NumberFormat
|
|
678
|
-
|
|
679
|
-
nDec: "0",
|
|
680
|
-
UseThou: "0",
|
|
681
|
-
Fmt: "",
|
|
682
|
-
Dec: "",
|
|
683
|
-
Thou: ""
|
|
684
|
-
},
|
|
685
|
-
Tags: existingField?.Tags || {}
|
|
1308
|
+
NumberFormat: resetContradictedNumberFormat(existingField?.NumberFormat, this._symbolFacts?.[index]),
|
|
1309
|
+
Tags: pruneContradictedTags(existingField?.Tags, this._symbolFacts?.[index])
|
|
686
1310
|
};
|
|
687
1311
|
})
|
|
688
1312
|
},
|
|
@@ -692,6 +1316,19 @@ var init_QvdFileWriter = __esm({
|
|
|
692
1316
|
Length: this._indexBuffer?.length
|
|
693
1317
|
}
|
|
694
1318
|
};
|
|
1319
|
+
const { Fields, ...table } = xmlObject.QvdTableHeader;
|
|
1320
|
+
for (const [property, value] of Object.entries(table)) {
|
|
1321
|
+
checkHeaderTexts(value, property, "the table", { file: this._path, stage: "buildHeader" });
|
|
1322
|
+
}
|
|
1323
|
+
for (const { FieldName, ...field } of Fields.QvdFieldHeader) {
|
|
1324
|
+
for (const [property, value] of Object.entries(field)) {
|
|
1325
|
+
checkHeaderTexts(value, property, `field '${FieldName}'`, {
|
|
1326
|
+
column: FieldName,
|
|
1327
|
+
file: this._path,
|
|
1328
|
+
stage: "buildHeader"
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
695
1332
|
const builder = new xml2.Builder({
|
|
696
1333
|
renderOpts: {
|
|
697
1334
|
pretty: true,
|
|
@@ -705,25 +1342,49 @@ var init_QvdFileWriter = __esm({
|
|
|
705
1342
|
/**
|
|
706
1343
|
* Builds the symbol table of the QVD file.
|
|
707
1344
|
*
|
|
708
|
-
*
|
|
709
|
-
*
|
|
710
|
-
*
|
|
1345
|
+
* One pass over the rows finds each column's distinct values in the order they first appear,
|
|
1346
|
+
* which is the order Qlik lists symbols in, and checks each distinct value once. A second pass per
|
|
1347
|
+
* column encodes them: every symbol is sized first, then written into one buffer of exactly that
|
|
1348
|
+
* size.
|
|
1349
|
+
*
|
|
1350
|
+
* What each value is stored as:
|
|
711
1351
|
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
*
|
|
716
|
-
*
|
|
1352
|
+
* | Cell | Symbol |
|
|
1353
|
+
* | --- | --- |
|
|
1354
|
+
* | `null`, `undefined`, a hole, a missing cell | none - the field's `Bias` records NULL |
|
|
1355
|
+
* | an integer from -2147483648 to 2147483647, -0 included | pure int, type 1 |
|
|
1356
|
+
* | any other finite number | pure double, type 2 |
|
|
1357
|
+
* | a string | pure string, type 4 |
|
|
1358
|
+
* | a dual - a `QvdDual`, or an object whose only keys are `number` and `text` | dual int or dual double, type 5 or 6, by the number |
|
|
1359
|
+
* | a number or a string the frame's `storedSymbols` records | the symbol it was read from |
|
|
717
1360
|
*
|
|
718
|
-
*
|
|
719
|
-
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
1361
|
+
* A number is a pure number, with no text. It used to be written as a dual whose text was
|
|
1362
|
+
* `String(value)`, which was wrong twice over: it invented text the caller never supplied, and a
|
|
1363
|
+
* file Qlik wrote with pure numbers came back out of a read and a write with every one of them
|
|
1364
|
+
* turned into a dual. The kind follows `isStoredAsInt`, which is the rule Qlik's own files follow.
|
|
1365
|
+
* A string is never parsed, so `'7'` and `7` in one column are two symbols. It is stored as its
|
|
1366
|
+
* UTF-8 bytes, or refused where those bytes would not give it back: a NUL ends a stored text, and
|
|
1367
|
+
* an unpaired surrogate has no UTF-8 encoding at all.
|
|
1368
|
+
*
|
|
1369
|
+
* A column holds one symbol per number, as a Qlik field does. Several duals with one number are one
|
|
1370
|
+
* symbol with the first text in row order, and a plain number with the same number as a dual joins
|
|
1371
|
+
* the dual, in either order - so no text a caller supplied is lost to a plain number that came
|
|
1372
|
+
* first. A string is not a number, so a string equal to a dual's text is a symbol of its own.
|
|
1373
|
+
*
|
|
1374
|
+
* A frame read from a file shows one half of some symbols: a dual read as its number or its text, a
|
|
1375
|
+
* string read as a number. Its record says what each such cell stands for, so the frame writes back
|
|
1376
|
+
* the symbols it was read from - the dual's text, Qlik's exact double, the string `'007'` - whichever
|
|
1377
|
+
* rows the cells were moved to. A cell the record maps to more than one stored value is refused.
|
|
1378
|
+
*
|
|
1379
|
+
* The complexity claim is the one to trust here: O(rows x columns) for the pass, and O(symbols)
|
|
1380
|
+
* for the encoding. The "80-90% improvement" this comment once carried is not reproducible in this
|
|
1381
|
+
* repository; `benchmarks/` measures what the writer costs now, which is the useful number.
|
|
722
1382
|
*
|
|
723
1383
|
* @private
|
|
724
1384
|
*/
|
|
725
1385
|
_buildSymbolTable() {
|
|
726
|
-
this.
|
|
1386
|
+
this._symbolCounts = [];
|
|
1387
|
+
this._symbolFacts = [];
|
|
727
1388
|
this._symbolTableMetadata = [];
|
|
728
1389
|
this._symbolIndexByValue = [];
|
|
729
1390
|
if (this._df.columns.length === 0) {
|
|
@@ -736,33 +1397,144 @@ var init_QvdFileWriter = __esm({
|
|
|
736
1397
|
const data = this._df.data;
|
|
737
1398
|
const numColumns = columns.length;
|
|
738
1399
|
const numRows = data.length;
|
|
1400
|
+
validateColumnNames(columns, this._path);
|
|
1401
|
+
const record = normaliseStoredSymbols(this._df.storedSymbols);
|
|
739
1402
|
this._emitProgress("symbol-table", 0, numColumns);
|
|
740
|
-
const
|
|
1403
|
+
const state = columns.map((name) => {
|
|
1404
|
+
const entry = record === null ? null : record.find((candidate) => candidate.field === name) ?? null;
|
|
1405
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1406
|
+
let byValue = null;
|
|
1407
|
+
let textByNumber = null;
|
|
1408
|
+
if (entry !== null && valuesAreNumbers(entry)) {
|
|
1409
|
+
textByNumber = firstTextByValue(entry);
|
|
1410
|
+
} else if (entry !== null) {
|
|
1411
|
+
byValue = /* @__PURE__ */ new Map();
|
|
1412
|
+
for (let index = 0; index < entry.values.length; index++) {
|
|
1413
|
+
const found = byValue.get(entry.values[index]);
|
|
1414
|
+
if (found === void 0) {
|
|
1415
|
+
byValue.set(entry.values[index], index);
|
|
1416
|
+
} else if (typeof found === "number") {
|
|
1417
|
+
byValue.set(entry.values[index], [found, index]);
|
|
1418
|
+
} else {
|
|
1419
|
+
found.push(index);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
return {
|
|
1424
|
+
name,
|
|
1425
|
+
keys: [],
|
|
1426
|
+
texts: [],
|
|
1427
|
+
byKey,
|
|
1428
|
+
// A cell is its own key without a record entry, and with one whose values are their numbers, so
|
|
1429
|
+
// one Map serves both. On a field of distinct duals that is a map entry per symbol fewer.
|
|
1430
|
+
byCell: entry === null || textByNumber !== null ? byKey : /* @__PURE__ */ new Map(),
|
|
1431
|
+
byObject: /* @__PURE__ */ new Map(),
|
|
1432
|
+
entry,
|
|
1433
|
+
textByNumber,
|
|
1434
|
+
byValue,
|
|
1435
|
+
facts: { hasNumber: false, hasNonNumber: false, hasFraction: false }
|
|
1436
|
+
};
|
|
1437
|
+
});
|
|
1438
|
+
const byCells = state.map((column) => column.byCell);
|
|
1439
|
+
const byObjects = state.map((column) => column.byObject);
|
|
741
1440
|
const containsNull = columns.map(() => false);
|
|
742
1441
|
for (let row = 0; row < numRows; row++) {
|
|
743
1442
|
const values = data[row];
|
|
1443
|
+
if (values !== null && values !== void 0 && !Array.isArray(values)) {
|
|
1444
|
+
throw new QvdValidationError("Each row must be an array of values", {
|
|
1445
|
+
row,
|
|
1446
|
+
type: typeof values,
|
|
1447
|
+
file: this._path,
|
|
1448
|
+
stage: "buildSymbolTable"
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1451
|
+
if (values !== null && values !== void 0 && values.length > numColumns) {
|
|
1452
|
+
throw new QvdValidationError(`Row ${row} has ${values.length} values but there are ${numColumns} fields`, {
|
|
1453
|
+
row,
|
|
1454
|
+
values: values.length,
|
|
1455
|
+
fields: numColumns,
|
|
1456
|
+
file: this._path,
|
|
1457
|
+
stage: "buildSymbolTable"
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
744
1460
|
for (let column = 0; column < numColumns; column++) {
|
|
745
1461
|
const value = values?.[column];
|
|
746
1462
|
if (value === null || value === void 0) {
|
|
747
1463
|
containsNull[column] = true;
|
|
748
1464
|
continue;
|
|
749
1465
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
1466
|
+
if (typeof value === "object") {
|
|
1467
|
+
if (!byObjects[column].has(value)) {
|
|
1468
|
+
slotForObject(state[column], value, row, this._path);
|
|
1469
|
+
}
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1472
|
+
if (byCells[column].has(value)) {
|
|
1473
|
+
continue;
|
|
1474
|
+
}
|
|
1475
|
+
if (typeof value === "number") {
|
|
1476
|
+
if (numberProblem(value) !== null) {
|
|
1477
|
+
checkNumber(value, null, { column: columns[column], row, file: this._path, stage: "buildSymbolTable" });
|
|
1478
|
+
}
|
|
1479
|
+
} else if (typeof value === "string") {
|
|
1480
|
+
if (textProblem(value) !== null) {
|
|
1481
|
+
checkText(value, "A string value", {
|
|
1482
|
+
column: columns[column],
|
|
1483
|
+
row,
|
|
1484
|
+
file: this._path,
|
|
1485
|
+
stage: "buildSymbolTable"
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
} else {
|
|
1489
|
+
refuseCell(value, columns[column], row, this._path);
|
|
753
1490
|
}
|
|
1491
|
+
byCells[column].set(value, slotForCell(state[column], value, row, this._path));
|
|
754
1492
|
}
|
|
755
1493
|
}
|
|
756
1494
|
const columnBuffers = [];
|
|
757
1495
|
let symbolsOffset = 0;
|
|
758
1496
|
for (let column = 0; column < numColumns; column++) {
|
|
759
|
-
const
|
|
760
|
-
const
|
|
1497
|
+
const { keys, texts } = state[column];
|
|
1498
|
+
const kinds = new Uint8Array(keys.length);
|
|
1499
|
+
let byteLength = 0;
|
|
1500
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1501
|
+
const key = keys[slot];
|
|
1502
|
+
const text = texts[slot];
|
|
1503
|
+
if (typeof key === "number") {
|
|
1504
|
+
if (numberProblem(key) !== null) {
|
|
1505
|
+
checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
|
|
1506
|
+
}
|
|
1507
|
+
if (text !== null && textProblem(text) !== null) {
|
|
1508
|
+
checkText(text, "The text of a dual value", {
|
|
1509
|
+
column: columns[column],
|
|
1510
|
+
file: this._path,
|
|
1511
|
+
stage: "buildSymbolTable"
|
|
1512
|
+
});
|
|
1513
|
+
}
|
|
1514
|
+
kinds[slot] = kindOf(key, text);
|
|
1515
|
+
byteLength += symbolByteLength(kinds[slot], key, text);
|
|
1516
|
+
} else {
|
|
1517
|
+
kinds[slot] = kindOf(null, key);
|
|
1518
|
+
byteLength += symbolByteLength(kinds[slot], null, key);
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
const columnBuffer = Buffer.allocUnsafe(byteLength);
|
|
1522
|
+
let offset = 0;
|
|
1523
|
+
for (let slot = 0; slot < keys.length; slot++) {
|
|
1524
|
+
const key = keys[slot];
|
|
1525
|
+
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1526
|
+
}
|
|
1527
|
+
assert2(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
761
1528
|
columnBuffers.push(columnBuffer);
|
|
762
|
-
this._symbolTableMetadata?.push([symbolsOffset,
|
|
763
|
-
this.
|
|
764
|
-
this.
|
|
765
|
-
|
|
1529
|
+
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1530
|
+
this._symbolCounts?.push(keys.length);
|
|
1531
|
+
this._symbolFacts?.push(state[column].facts);
|
|
1532
|
+
this._symbolIndexByValue?.push({
|
|
1533
|
+
byCell: state[column].byCell,
|
|
1534
|
+
byKey: state[column].byKey,
|
|
1535
|
+
byObject: state[column].byObject
|
|
1536
|
+
});
|
|
1537
|
+
symbolsOffset += byteLength;
|
|
766
1538
|
this._emitProgress("symbol-table", column + 1, numColumns);
|
|
767
1539
|
}
|
|
768
1540
|
this._symbolBuffer = Buffer.concat(columnBuffers);
|
|
@@ -791,7 +1563,7 @@ var init_QvdFileWriter = __esm({
|
|
|
791
1563
|
* @private
|
|
792
1564
|
*/
|
|
793
1565
|
_buildIndexTable() {
|
|
794
|
-
assert2(this.
|
|
1566
|
+
assert2(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
795
1567
|
assert2(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
796
1568
|
assert2(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
797
1569
|
this._indexTableMetadata = [];
|
|
@@ -805,20 +1577,24 @@ var init_QvdFileWriter = __esm({
|
|
|
805
1577
|
let totalBits = 0;
|
|
806
1578
|
for (let column = 0; column < numColumns; column++) {
|
|
807
1579
|
const fieldContainsNull = this._symbolTableMetadata[column][2];
|
|
808
|
-
const symbolCount = this.
|
|
1580
|
+
const symbolCount = this._symbolCounts[column];
|
|
809
1581
|
const nullShift = fieldContainsNull ? 2 : 0;
|
|
810
1582
|
const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
|
|
811
1583
|
const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
1584
|
+
const lookup = this._symbolIndexByValue[column];
|
|
1585
|
+
const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
|
|
1586
|
+
for (const map of maps) {
|
|
1587
|
+
for (const index of map.values()) {
|
|
1588
|
+
if (index + nullShift > maxStoredIndex) {
|
|
1589
|
+
throw new QvdValidationError("The symbol table and the index table are out of sync", {
|
|
1590
|
+
field: columns[column],
|
|
1591
|
+
storedIndex: index + nullShift,
|
|
1592
|
+
maxStoredIndex,
|
|
1593
|
+
symbolCount,
|
|
1594
|
+
file: this._path,
|
|
1595
|
+
stage: "buildIndexTable"
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
822
1598
|
}
|
|
823
1599
|
}
|
|
824
1600
|
layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
|
|
@@ -829,6 +1605,9 @@ var init_QvdFileWriter = __esm({
|
|
|
829
1605
|
this._recordByteSize = recordByteSize;
|
|
830
1606
|
this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
|
|
831
1607
|
const progressInterval = Math.max(1, Math.floor(numRows / 100));
|
|
1608
|
+
const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
|
|
1609
|
+
const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
|
|
1610
|
+
const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
|
|
832
1611
|
for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
|
|
833
1612
|
const values = data[row];
|
|
834
1613
|
for (let column = 0; column < numColumns; column++) {
|
|
@@ -836,7 +1615,7 @@ var init_QvdFileWriter = __esm({
|
|
|
836
1615
|
if (value === null || value === void 0) {
|
|
837
1616
|
continue;
|
|
838
1617
|
}
|
|
839
|
-
const index =
|
|
1618
|
+
const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
|
|
840
1619
|
if (index === void 0) {
|
|
841
1620
|
throw new QvdValidationError("A value is missing from the symbol table", {
|
|
842
1621
|
field: columns[column],
|
|
@@ -853,29 +1632,6 @@ var init_QvdFileWriter = __esm({
|
|
|
853
1632
|
}
|
|
854
1633
|
this._symbolIndexByValue = null;
|
|
855
1634
|
}
|
|
856
|
-
/**
|
|
857
|
-
* Converts a raw value/literal to a QVD symbol.
|
|
858
|
-
*
|
|
859
|
-
* @param {any} raw The raw value/literal to convert.
|
|
860
|
-
* @return {QvdSymbol|null} The converted QVD symbol.
|
|
861
|
-
*/
|
|
862
|
-
static _convertRawToSymbol(raw) {
|
|
863
|
-
if (raw === null || raw === void 0) {
|
|
864
|
-
return null;
|
|
865
|
-
}
|
|
866
|
-
const INT32_MIN = -2147483648;
|
|
867
|
-
const INT32_MAX = 2147483647;
|
|
868
|
-
const isInteger = typeof raw === "number" && Number.isInteger(raw);
|
|
869
|
-
const isFloat = typeof raw === "number" && !Number.isInteger(raw);
|
|
870
|
-
const isWithinInt32Range = typeof raw === "number" && raw >= INT32_MIN && raw <= INT32_MAX;
|
|
871
|
-
if (isInteger && isWithinInt32Range) {
|
|
872
|
-
return QvdSymbol.fromDualIntValue(raw, raw.toString());
|
|
873
|
-
} else if (isFloat || isInteger && !isWithinInt32Range) {
|
|
874
|
-
return QvdSymbol.fromDualDoubleValue(raw, raw.toString());
|
|
875
|
-
} else {
|
|
876
|
-
return QvdSymbol.fromStringValue(raw);
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
1635
|
/**
|
|
880
1636
|
* Persists the data frame to a QVD file.
|
|
881
1637
|
*/
|
|
@@ -945,7 +1701,7 @@ function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
945
1701
|
return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
|
|
946
1702
|
}
|
|
947
1703
|
function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true, includeExternal = false) {
|
|
948
|
-
const costOf = (rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0);
|
|
1704
|
+
const costOf = /* @__PURE__ */ __name((rows) => estimateMemoryUsage(symbolTableSize, rows, totalRows, columnCount, materialisesRows) + (includeExternal ? estimateExternalMemory(Math.min(rows, totalRows), columnCount) : 0), "costOf");
|
|
949
1705
|
if (costOf(totalRows) <= budget) {
|
|
950
1706
|
return totalRows;
|
|
951
1707
|
}
|
|
@@ -966,11 +1722,11 @@ function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, mat
|
|
|
966
1722
|
}
|
|
967
1723
|
function recommendedChunkFor(budget, symbolTableSize, windowRows, totalRows, columnCount, liveRowsPerChunk = 1, includeExternal = false) {
|
|
968
1724
|
const covered = windowRows === null || windowRows >= totalRows ? totalRows : windowRows;
|
|
969
|
-
const fits = (chunk) => {
|
|
1725
|
+
const fits = /* @__PURE__ */ __name((chunk) => {
|
|
970
1726
|
const live = Math.min(chunk * liveRowsPerChunk, covered);
|
|
971
1727
|
const cost = estimateMemoryUsage(symbolTableSize, windowRows, totalRows, columnCount, true, chunk * liveRowsPerChunk) + (includeExternal ? estimateExternalMemory(live, columnCount) : 0);
|
|
972
1728
|
return cost <= budget;
|
|
973
|
-
};
|
|
1729
|
+
}, "fits");
|
|
974
1730
|
if (fits(covered)) {
|
|
975
1731
|
return covered;
|
|
976
1732
|
}
|
|
@@ -1060,9 +1816,9 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
1060
1816
|
if (nothingFits) {
|
|
1061
1817
|
advice = `No row count fits this budget - the symbol table alone exceeds it, so ${knob} cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
|
|
1062
1818
|
} else if (containerBound) {
|
|
1063
|
-
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${recommendedValue
|
|
1819
|
+
advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or hold fewer rows with ${knob} (recommended: ${formatCount(recommendedValue)} rows or less).`;
|
|
1064
1820
|
} else {
|
|
1065
|
-
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${recommendedValue
|
|
1821
|
+
advice = `Try holding fewer rows using the ${knob} parameter (recommended: ${formatCount(recommendedValue)} rows or less), or raise the heap with --max-old-space-size.`;
|
|
1066
1822
|
}
|
|
1067
1823
|
throw new QvdValidationError(
|
|
1068
1824
|
`Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
|
|
@@ -1090,6 +1846,9 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
|
|
|
1090
1846
|
);
|
|
1091
1847
|
}
|
|
1092
1848
|
}
|
|
1849
|
+
function formatCount(value) {
|
|
1850
|
+
return value.toLocaleString("en-US");
|
|
1851
|
+
}
|
|
1093
1852
|
function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
|
|
1094
1853
|
const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
|
|
1095
1854
|
if (symbolTableSize <= LARGE_SYMBOL_TABLE_WARNING) {
|
|
@@ -1104,18 +1863,30 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount =
|
|
|
1104
1863
|
const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
|
|
1105
1864
|
const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
|
|
1106
1865
|
console.warn(
|
|
1107
|
-
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${rowsToLoad
|
|
1866
|
+
`\u26A0\uFE0F Large symbol table detected (${sizeMB}MB > ${warnMB}MB threshold). This read materialises ${formatCount(rowsToLoad)} of ${formatCount(totalRows)} rows and will use ~${estimatedMB}MB RAM. Reading fewer rows - with limit, maxRows, or a narrower offset window - lowers the row cost, though the symbol table is read in full either way.`
|
|
1108
1867
|
);
|
|
1109
1868
|
}
|
|
1110
1869
|
var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
|
|
1111
1870
|
var init_memoryUtils = __esm({
|
|
1112
1871
|
"src/util/memoryUtils.js"() {
|
|
1113
1872
|
init_QvdErrors();
|
|
1873
|
+
__name(getHeapLimit, "getHeapLimit");
|
|
1874
|
+
__name(heapLimitIsMeaningful, "heapLimitIsMeaningful");
|
|
1875
|
+
__name(getMemoryBudget, "getMemoryBudget");
|
|
1114
1876
|
HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
|
|
1115
1877
|
MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
|
|
1878
|
+
__name(usableOldSpaceLimit, "usableOldSpaceLimit");
|
|
1116
1879
|
BASE_BYTES = 16 * 1024 * 1024;
|
|
1117
1880
|
ROW_BASE_BYTES = 72;
|
|
1118
1881
|
PER_CELL_BYTES = 8;
|
|
1882
|
+
__name(estimateExternalMemory, "estimateExternalMemory");
|
|
1883
|
+
__name(estimateRowMemory, "estimateRowMemory");
|
|
1884
|
+
__name(estimateMemoryUsage, "estimateMemoryUsage");
|
|
1885
|
+
__name(recommendedRowsFor, "recommendedRowsFor");
|
|
1886
|
+
__name(recommendedChunkFor, "recommendedChunkFor");
|
|
1887
|
+
__name(validateMemoryAvailability, "validateMemoryAvailability");
|
|
1888
|
+
__name(formatCount, "formatCount");
|
|
1889
|
+
__name(warnLargeSymbolTable, "warnLargeSymbolTable");
|
|
1119
1890
|
}
|
|
1120
1891
|
});
|
|
1121
1892
|
|
|
@@ -1352,274 +2123,247 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
1352
2123
|
throw new QvdCorruptedError("Bit field extends beyond record size", {
|
|
1353
2124
|
field: field["FieldName"],
|
|
1354
2125
|
bitOffset,
|
|
1355
|
-
bitWidth,
|
|
1356
|
-
recordSizeInBits,
|
|
1357
|
-
file: filePath,
|
|
1358
|
-
stage: "parseIndexTable"
|
|
1359
|
-
});
|
|
1360
|
-
}
|
|
1361
|
-
}
|
|
1362
|
-
var init_validationUtils = __esm({
|
|
1363
|
-
"src/util/validationUtils.js"() {
|
|
1364
|
-
init_QvdErrors();
|
|
1365
|
-
init_memoryUtils();
|
|
1366
|
-
init_bitUtils();
|
|
1367
|
-
}
|
|
1368
|
-
});
|
|
1369
|
-
|
|
1370
|
-
// src/util/symbolParser.js
|
|
1371
|
-
function parseIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1372
|
-
if (pointer + 4 > bufferLength) {
|
|
1373
|
-
throw new QvdCorruptedError("Buffer overflow reading integer symbol", {
|
|
1374
|
-
field: fieldName,
|
|
1375
|
-
pointer,
|
|
1376
|
-
bufferSize: bufferLength,
|
|
1377
|
-
file: filePath,
|
|
1378
|
-
stage: "parseSymbolTable"
|
|
1379
|
-
});
|
|
1380
|
-
}
|
|
1381
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1382
|
-
const value = Buffer.from(byteData).readIntLE(0, byteData.length);
|
|
1383
|
-
return { symbol: QvdSymbol.fromIntValue(value), bytesRead: 4 };
|
|
1384
|
-
}
|
|
1385
|
-
function parseDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1386
|
-
if (pointer + 8 > bufferLength) {
|
|
1387
|
-
throw new QvdCorruptedError("Buffer overflow reading double symbol", {
|
|
1388
|
-
field: fieldName,
|
|
1389
|
-
pointer,
|
|
1390
|
-
bufferSize: bufferLength,
|
|
1391
|
-
file: filePath,
|
|
1392
|
-
stage: "parseSymbolTable"
|
|
1393
|
-
});
|
|
1394
|
-
}
|
|
1395
|
-
const byteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1396
|
-
const value = Buffer.from(byteData).readDoubleLE(0);
|
|
1397
|
-
return { symbol: QvdSymbol.fromDoubleValue(value), bytesRead: 8 };
|
|
1398
|
-
}
|
|
1399
|
-
function parseStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1400
|
-
const startPointer = pointer;
|
|
1401
|
-
const maxStringLength = 1048576;
|
|
1402
|
-
let stringLength = 0;
|
|
1403
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1404
|
-
if (stringLength >= maxStringLength) {
|
|
1405
|
-
throw new QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1406
|
-
field: fieldName,
|
|
1407
|
-
maxLength: maxStringLength,
|
|
1408
|
-
file: filePath,
|
|
1409
|
-
stage: "parseSymbolTable"
|
|
1410
|
-
});
|
|
1411
|
-
}
|
|
1412
|
-
pointer++;
|
|
1413
|
-
stringLength++;
|
|
1414
|
-
}
|
|
1415
|
-
if (pointer >= bufferLength) {
|
|
1416
|
-
throw new QvdCorruptedError("String symbol not null-terminated", {
|
|
1417
|
-
field: fieldName,
|
|
1418
|
-
pointer,
|
|
1419
|
-
bufferSize: bufferLength,
|
|
1420
|
-
file: filePath,
|
|
1421
|
-
stage: "parseSymbolTable"
|
|
1422
|
-
});
|
|
1423
|
-
}
|
|
1424
|
-
const value = symbolBuffer.subarray(startPointer, pointer).toString("utf-8");
|
|
1425
|
-
return { symbol: QvdSymbol.fromStringValue(value), bytesRead: pointer - startPointer + 1 };
|
|
1426
|
-
}
|
|
1427
|
-
function skipStringSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1428
|
-
const startPointer = pointer;
|
|
1429
|
-
const maxStringLength = 1048576;
|
|
1430
|
-
let stringLength = 0;
|
|
1431
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1432
|
-
if (stringLength >= maxStringLength) {
|
|
1433
|
-
throw new QvdCorruptedError("String symbol exceeds maximum length", {
|
|
1434
|
-
field: fieldName,
|
|
1435
|
-
maxLength: maxStringLength,
|
|
1436
|
-
file: filePath,
|
|
1437
|
-
stage: "parseSymbolTable"
|
|
1438
|
-
});
|
|
1439
|
-
}
|
|
1440
|
-
pointer++;
|
|
1441
|
-
stringLength++;
|
|
1442
|
-
}
|
|
1443
|
-
if (pointer >= bufferLength) {
|
|
1444
|
-
throw new QvdCorruptedError("String symbol not null-terminated", {
|
|
1445
|
-
field: fieldName,
|
|
1446
|
-
pointer,
|
|
1447
|
-
bufferSize: bufferLength,
|
|
1448
|
-
file: filePath,
|
|
1449
|
-
stage: "parseSymbolTable"
|
|
1450
|
-
});
|
|
1451
|
-
}
|
|
1452
|
-
return pointer - startPointer + 1;
|
|
1453
|
-
}
|
|
1454
|
-
function parseDualIntegerSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1455
|
-
if (pointer + 4 > bufferLength) {
|
|
1456
|
-
throw new QvdCorruptedError("Buffer overflow reading dual integer symbol", {
|
|
1457
|
-
field: fieldName,
|
|
1458
|
-
pointer,
|
|
1459
|
-
bufferSize: bufferLength,
|
|
1460
|
-
file: filePath,
|
|
1461
|
-
stage: "parseSymbolTable"
|
|
1462
|
-
});
|
|
1463
|
-
}
|
|
1464
|
-
const intByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 4));
|
|
1465
|
-
const intValue = Buffer.from(intByteData).readIntLE(0, intByteData.length);
|
|
1466
|
-
pointer += 4;
|
|
1467
|
-
const stringStart = pointer;
|
|
1468
|
-
const maxStringLength = 1048576;
|
|
1469
|
-
let stringLength = 0;
|
|
1470
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1471
|
-
if (stringLength >= maxStringLength) {
|
|
1472
|
-
throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1473
|
-
field: fieldName,
|
|
1474
|
-
maxLength: maxStringLength,
|
|
1475
|
-
file: filePath,
|
|
1476
|
-
stage: "parseSymbolTable"
|
|
1477
|
-
});
|
|
1478
|
-
}
|
|
1479
|
-
pointer++;
|
|
1480
|
-
stringLength++;
|
|
1481
|
-
}
|
|
1482
|
-
if (pointer >= bufferLength) {
|
|
1483
|
-
throw new QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1484
|
-
field: fieldName,
|
|
1485
|
-
pointer,
|
|
1486
|
-
bufferSize: bufferLength,
|
|
1487
|
-
file: filePath,
|
|
1488
|
-
stage: "parseSymbolTable"
|
|
1489
|
-
});
|
|
1490
|
-
}
|
|
1491
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1492
|
-
return { symbol: QvdSymbol.fromDualIntValue(intValue, stringValue), bytesRead: pointer - (stringStart - 4) + 1 };
|
|
1493
|
-
}
|
|
1494
|
-
function parseDualDoubleSymbol(symbolBuffer, pointer, bufferLength, fieldName, filePath) {
|
|
1495
|
-
if (pointer + 8 > bufferLength) {
|
|
1496
|
-
throw new QvdCorruptedError("Buffer overflow reading dual double symbol", {
|
|
1497
|
-
field: fieldName,
|
|
1498
|
-
pointer,
|
|
1499
|
-
bufferSize: bufferLength,
|
|
1500
|
-
file: filePath,
|
|
1501
|
-
stage: "parseSymbolTable"
|
|
1502
|
-
});
|
|
1503
|
-
}
|
|
1504
|
-
const doubleByteData = new Int32Array(symbolBuffer.subarray(pointer, pointer + 8));
|
|
1505
|
-
const doubleValue = Buffer.from(doubleByteData).readDoubleLE(0);
|
|
1506
|
-
pointer += 8;
|
|
1507
|
-
const stringStart = pointer;
|
|
1508
|
-
const maxStringLength = 1048576;
|
|
1509
|
-
let stringLength = 0;
|
|
1510
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1511
|
-
if (stringLength >= maxStringLength) {
|
|
1512
|
-
throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1513
|
-
field: fieldName,
|
|
1514
|
-
maxLength: maxStringLength,
|
|
1515
|
-
file: filePath,
|
|
1516
|
-
stage: "parseSymbolTable"
|
|
1517
|
-
});
|
|
1518
|
-
}
|
|
1519
|
-
pointer++;
|
|
1520
|
-
stringLength++;
|
|
1521
|
-
}
|
|
1522
|
-
if (pointer >= bufferLength) {
|
|
1523
|
-
throw new QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
1524
|
-
field: fieldName,
|
|
1525
|
-
pointer,
|
|
1526
|
-
bufferSize: bufferLength,
|
|
2126
|
+
bitWidth,
|
|
2127
|
+
recordSizeInBits,
|
|
1527
2128
|
file: filePath,
|
|
1528
|
-
stage: "
|
|
2129
|
+
stage: "parseIndexTable"
|
|
1529
2130
|
});
|
|
1530
2131
|
}
|
|
1531
|
-
const stringValue = symbolBuffer.subarray(stringStart, pointer).toString("utf-8");
|
|
1532
|
-
return {
|
|
1533
|
-
symbol: QvdSymbol.fromDualDoubleValue(doubleValue, stringValue),
|
|
1534
|
-
bytesRead: pointer - (stringStart - 8) + 1
|
|
1535
|
-
};
|
|
1536
2132
|
}
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
2133
|
+
var init_validationUtils = __esm({
|
|
2134
|
+
"src/util/validationUtils.js"() {
|
|
2135
|
+
init_QvdErrors();
|
|
2136
|
+
init_memoryUtils();
|
|
2137
|
+
init_bitUtils();
|
|
2138
|
+
__name(validateHeaderStructure, "validateHeaderStructure");
|
|
2139
|
+
__name(validateSymbolTableSizeEarly, "validateSymbolTableSizeEarly");
|
|
2140
|
+
__name(validateSymbolTableSize, "validateSymbolTableSize");
|
|
2141
|
+
__name(validateFieldMetadata, "validateFieldMetadata");
|
|
2142
|
+
__name(validateRecordCount, "validateRecordCount");
|
|
2143
|
+
__name(validateIndexTableMetadata, "validateIndexTableMetadata");
|
|
2144
|
+
__name(validateFieldBitMetadata, "validateFieldBitMetadata");
|
|
2145
|
+
}
|
|
2146
|
+
});
|
|
2147
|
+
|
|
2148
|
+
// src/util/symbolParser.js
|
|
2149
|
+
function textEnd(symbolBuffer, from, kind, fieldName, filePath) {
|
|
2150
|
+
const bufferLength = symbolBuffer.length;
|
|
2151
|
+
const found = symbolBuffer.indexOf(0, from);
|
|
2152
|
+
if ((found === -1 ? bufferLength : found) - from > MAX_TEXT_BYTES) {
|
|
2153
|
+
throw new QvdCorruptedError(`${kind} exceeds maximum length`, {
|
|
1540
2154
|
field: fieldName,
|
|
1541
|
-
|
|
1542
|
-
bufferSize: bufferLength,
|
|
2155
|
+
maxLength: MAX_TEXT_BYTES,
|
|
1543
2156
|
file: filePath,
|
|
1544
2157
|
stage: "parseSymbolTable"
|
|
1545
2158
|
});
|
|
1546
2159
|
}
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
const maxStringLength = 1048576;
|
|
1550
|
-
let stringLength = 0;
|
|
1551
|
-
while (pointer < bufferLength && symbolBuffer[pointer] !== 0) {
|
|
1552
|
-
if (stringLength >= maxStringLength) {
|
|
1553
|
-
throw new QvdCorruptedError("Dual string symbol exceeds maximum length", {
|
|
1554
|
-
field: fieldName,
|
|
1555
|
-
maxLength: maxStringLength,
|
|
1556
|
-
file: filePath,
|
|
1557
|
-
stage: "parseSymbolTable"
|
|
1558
|
-
});
|
|
1559
|
-
}
|
|
1560
|
-
pointer++;
|
|
1561
|
-
stringLength++;
|
|
1562
|
-
}
|
|
1563
|
-
if (pointer >= bufferLength) {
|
|
1564
|
-
throw new QvdCorruptedError("Dual string symbol not null-terminated", {
|
|
2160
|
+
if (found === -1) {
|
|
2161
|
+
throw new QvdCorruptedError(`${kind} not null-terminated`, {
|
|
1565
2162
|
field: fieldName,
|
|
1566
|
-
pointer,
|
|
2163
|
+
pointer: bufferLength,
|
|
1567
2164
|
bufferSize: bufferLength,
|
|
1568
2165
|
file: filePath,
|
|
1569
2166
|
stage: "parseSymbolTable"
|
|
1570
2167
|
});
|
|
1571
2168
|
}
|
|
1572
|
-
return
|
|
2169
|
+
return found;
|
|
1573
2170
|
}
|
|
1574
|
-
function
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
2171
|
+
function overflow(message, pointer, bufferLength, fieldName, filePath) {
|
|
2172
|
+
throw new QvdCorruptedError(message, {
|
|
2173
|
+
field: fieldName,
|
|
2174
|
+
pointer,
|
|
2175
|
+
bufferSize: bufferLength,
|
|
2176
|
+
file: filePath,
|
|
2177
|
+
stage: "parseSymbolTable"
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath) {
|
|
2181
|
+
const bufferLength = symbolBuffer.length;
|
|
2182
|
+
const numbers = [];
|
|
2183
|
+
const texts = [];
|
|
2184
|
+
let pointer = start;
|
|
2185
|
+
while (pointer < end) {
|
|
2186
|
+
const typeByte = symbolBuffer[pointer++];
|
|
2187
|
+
const decode = keep === null || keep.has(numbers.length);
|
|
2188
|
+
let number = null;
|
|
2189
|
+
let text = null;
|
|
2190
|
+
switch (typeByte) {
|
|
2191
|
+
case 1: {
|
|
2192
|
+
if (decode) {
|
|
2193
|
+
if (pointer + 4 > bufferLength) {
|
|
2194
|
+
overflow("Buffer overflow reading integer symbol", pointer, bufferLength, fieldName, filePath);
|
|
2195
|
+
}
|
|
2196
|
+
number = symbolBuffer.readInt32LE(pointer);
|
|
2197
|
+
}
|
|
2198
|
+
pointer += 4;
|
|
2199
|
+
break;
|
|
1579
2200
|
}
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
2201
|
+
case 2: {
|
|
2202
|
+
if (decode) {
|
|
2203
|
+
if (pointer + 8 > bufferLength) {
|
|
2204
|
+
overflow("Buffer overflow reading double symbol", pointer, bufferLength, fieldName, filePath);
|
|
2205
|
+
}
|
|
2206
|
+
number = symbolBuffer.readDoubleLE(pointer);
|
|
2207
|
+
}
|
|
2208
|
+
pointer += 8;
|
|
2209
|
+
break;
|
|
1585
2210
|
}
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
2211
|
+
case 4: {
|
|
2212
|
+
const terminator = textEnd(symbolBuffer, pointer, "String symbol", fieldName, filePath);
|
|
2213
|
+
if (decode) {
|
|
2214
|
+
text = symbolBuffer.toString("utf8", pointer, terminator);
|
|
2215
|
+
}
|
|
2216
|
+
pointer = terminator + 1;
|
|
2217
|
+
break;
|
|
1591
2218
|
}
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
2219
|
+
case 5:
|
|
2220
|
+
case 6: {
|
|
2221
|
+
const numberBytes = typeByte === 5 ? 4 : 8;
|
|
2222
|
+
if (pointer + numberBytes > bufferLength) {
|
|
2223
|
+
const read = !decode ? "dual symbol" : typeByte === 5 ? "dual integer symbol" : "dual double symbol";
|
|
2224
|
+
overflow(`Buffer overflow reading ${read}`, pointer, bufferLength, fieldName, filePath);
|
|
2225
|
+
}
|
|
2226
|
+
const terminator = textEnd(symbolBuffer, pointer + numberBytes, "Dual string symbol", fieldName, filePath);
|
|
2227
|
+
if (decode) {
|
|
2228
|
+
number = typeByte === 5 ? symbolBuffer.readInt32LE(pointer) : symbolBuffer.readDoubleLE(pointer);
|
|
2229
|
+
text = symbolBuffer.toString("utf8", pointer + numberBytes, terminator);
|
|
2230
|
+
}
|
|
2231
|
+
pointer = terminator + 1;
|
|
2232
|
+
break;
|
|
1598
2233
|
}
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
2234
|
+
default: {
|
|
2235
|
+
throw new QvdParseError("Unknown symbol type byte", {
|
|
2236
|
+
typeByte: typeByte.toString(16),
|
|
2237
|
+
offset: pointer - 1,
|
|
2238
|
+
file: filePath,
|
|
2239
|
+
stage: "parseSymbolTable"
|
|
2240
|
+
});
|
|
1605
2241
|
}
|
|
1606
|
-
const bytesRead = skipDualSymbol(symbolBuffer, pointer, bufferLength, 8, fieldName, filePath);
|
|
1607
|
-
return { symbol: null, bytesRead };
|
|
1608
|
-
}
|
|
1609
|
-
default: {
|
|
1610
|
-
throw new QvdParseError("Unknown symbol type byte", {
|
|
1611
|
-
typeByte: typeByte.toString(16),
|
|
1612
|
-
offset: pointer - 1,
|
|
1613
|
-
file: filePath,
|
|
1614
|
-
stage: "parseSymbolTable"
|
|
1615
|
-
});
|
|
1616
2242
|
}
|
|
2243
|
+
numbers.push(number);
|
|
2244
|
+
texts.push(text);
|
|
1617
2245
|
}
|
|
2246
|
+
return { numbers, texts };
|
|
1618
2247
|
}
|
|
2248
|
+
var MAX_TEXT_BYTES;
|
|
1619
2249
|
var init_symbolParser = __esm({
|
|
1620
2250
|
"src/util/symbolParser.js"() {
|
|
1621
|
-
init_QvdSymbol();
|
|
1622
2251
|
init_QvdErrors();
|
|
2252
|
+
MAX_TEXT_BYTES = 1048576;
|
|
2253
|
+
__name(textEnd, "textEnd");
|
|
2254
|
+
__name(overflow, "overflow");
|
|
2255
|
+
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2256
|
+
}
|
|
2257
|
+
});
|
|
2258
|
+
|
|
2259
|
+
// src/util/resolveSymbols.js
|
|
2260
|
+
function resolveFieldSymbols(symbols, field, mode, coerce, wantHalves) {
|
|
2261
|
+
const length = symbols.numbers.length;
|
|
2262
|
+
const values = new Array(length);
|
|
2263
|
+
const entryValues = [];
|
|
2264
|
+
const numbers = [];
|
|
2265
|
+
const texts = [];
|
|
2266
|
+
let pure = 0;
|
|
2267
|
+
let partial = false;
|
|
2268
|
+
for (let index = 0; index < length; index++) {
|
|
2269
|
+
const text = symbols.texts[index];
|
|
2270
|
+
const number = symbols.numbers[index];
|
|
2271
|
+
if (text === null) {
|
|
2272
|
+
values[index] = number === null ? void 0 : number;
|
|
2273
|
+
if (number !== null) pure++;
|
|
2274
|
+
continue;
|
|
2275
|
+
}
|
|
2276
|
+
if (number === null) {
|
|
2277
|
+
if (coerce && isNumericText(text)) {
|
|
2278
|
+
values[index] = Number(text);
|
|
2279
|
+
entryValues.push(values[index]);
|
|
2280
|
+
numbers.push(null);
|
|
2281
|
+
texts.push(text);
|
|
2282
|
+
partial = true;
|
|
2283
|
+
} else {
|
|
2284
|
+
values[index] = text;
|
|
2285
|
+
pure++;
|
|
2286
|
+
}
|
|
2287
|
+
continue;
|
|
2288
|
+
}
|
|
2289
|
+
partial = true;
|
|
2290
|
+
if (mode === "both") {
|
|
2291
|
+
values[index] = dualFromSymbol(number, text);
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
values[index] = mode === "number" || coerce && isNumericText(text) ? number : text;
|
|
2295
|
+
entryValues.push(values[index]);
|
|
2296
|
+
numbers.push(number);
|
|
2297
|
+
texts.push(text);
|
|
2298
|
+
}
|
|
2299
|
+
let entry = null;
|
|
2300
|
+
if (entryValues.length > 0) {
|
|
2301
|
+
entry = pure > 0 && collides(symbols, values, new Set(entryValues)) ? collisionEntry(symbols, field, values, mode) : Object.freeze({
|
|
2302
|
+
field,
|
|
2303
|
+
values: Object.freeze(entryValues),
|
|
2304
|
+
numbers: Object.freeze(numbers),
|
|
2305
|
+
texts: Object.freeze(texts)
|
|
2306
|
+
});
|
|
2307
|
+
}
|
|
2308
|
+
return { values, entry, halves: wantHalves && partial ? symbolHalves(symbols) : null };
|
|
2309
|
+
}
|
|
2310
|
+
function collides(symbols, values, recorded) {
|
|
2311
|
+
for (let index = 0; index < values.length; index++) {
|
|
2312
|
+
if (!isPure(symbols, index, values[index])) {
|
|
2313
|
+
continue;
|
|
2314
|
+
}
|
|
2315
|
+
if (recorded.has(values[index])) {
|
|
2316
|
+
return true;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
return false;
|
|
2320
|
+
}
|
|
2321
|
+
function isPure(symbols, index, value) {
|
|
2322
|
+
const text = symbols.texts[index];
|
|
2323
|
+
const number = symbols.numbers[index];
|
|
2324
|
+
return text === null ? number !== null : number === null && value === text;
|
|
2325
|
+
}
|
|
2326
|
+
function collisionEntry(symbols, field, values, mode) {
|
|
2327
|
+
const recorded = /* @__PURE__ */ new Set();
|
|
2328
|
+
const inPassA = values.map((value, index) => {
|
|
2329
|
+
if (symbols.numbers[index] === null && symbols.texts[index] === null || isPure(symbols, index, value)) {
|
|
2330
|
+
return false;
|
|
2331
|
+
}
|
|
2332
|
+
const recordedHere = !(mode === "both" && symbols.texts[index] !== null && symbols.numbers[index] !== null);
|
|
2333
|
+
if (recordedHere) {
|
|
2334
|
+
recorded.add(values[index]);
|
|
2335
|
+
}
|
|
2336
|
+
return recordedHere;
|
|
2337
|
+
});
|
|
2338
|
+
const entryValues = [];
|
|
2339
|
+
const numbers = [];
|
|
2340
|
+
const texts = [];
|
|
2341
|
+
for (let index = 0; index < values.length; index++) {
|
|
2342
|
+
if (inPassA[index] || isPure(symbols, index, values[index]) && recorded.has(values[index])) {
|
|
2343
|
+
entryValues.push(values[index]);
|
|
2344
|
+
numbers.push(symbols.numbers[index]);
|
|
2345
|
+
texts.push(symbols.texts[index]);
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
return Object.freeze({
|
|
2349
|
+
field,
|
|
2350
|
+
values: Object.freeze(entryValues),
|
|
2351
|
+
numbers: Object.freeze(numbers),
|
|
2352
|
+
texts: Object.freeze(texts)
|
|
2353
|
+
});
|
|
2354
|
+
}
|
|
2355
|
+
function symbolHalves(symbols) {
|
|
2356
|
+
return Object.freeze({ texts: Object.freeze(symbols.texts), numbers: Object.freeze(symbols.numbers) });
|
|
2357
|
+
}
|
|
2358
|
+
var init_resolveSymbols = __esm({
|
|
2359
|
+
"src/util/resolveSymbols.js"() {
|
|
2360
|
+
init_QvdDual();
|
|
2361
|
+
init_cellRules();
|
|
2362
|
+
__name(resolveFieldSymbols, "resolveFieldSymbols");
|
|
2363
|
+
__name(collides, "collides");
|
|
2364
|
+
__name(isPure, "isPure");
|
|
2365
|
+
__name(collisionEntry, "collisionEntry");
|
|
2366
|
+
__name(symbolHalves, "symbolHalves");
|
|
1623
2367
|
}
|
|
1624
2368
|
});
|
|
1625
2369
|
|
|
@@ -1629,21 +2373,49 @@ __export(QvdColumnTable_exports, {
|
|
|
1629
2373
|
QvdColumn: () => QvdColumn,
|
|
1630
2374
|
QvdColumnTable: () => QvdColumnTable
|
|
1631
2375
|
});
|
|
2376
|
+
function numberOf(value, number) {
|
|
2377
|
+
if (typeof value === "number") {
|
|
2378
|
+
return value;
|
|
2379
|
+
}
|
|
2380
|
+
if (typeof value === "string") {
|
|
2381
|
+
return number ?? NaN;
|
|
2382
|
+
}
|
|
2383
|
+
const dual = asDual(value);
|
|
2384
|
+
return dual !== null && typeof dual.number === "number" ? dual.number : NaN;
|
|
2385
|
+
}
|
|
2386
|
+
function textOf(value) {
|
|
2387
|
+
if (typeof value === "string") {
|
|
2388
|
+
return value;
|
|
2389
|
+
}
|
|
2390
|
+
const dual = asDual(value);
|
|
2391
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
2392
|
+
}
|
|
1632
2393
|
var QvdColumn, QvdColumnTable;
|
|
1633
2394
|
var init_QvdColumnTable = __esm({
|
|
1634
2395
|
"src/QvdColumnTable.js"() {
|
|
1635
2396
|
init_QvdErrors();
|
|
2397
|
+
init_cellRules();
|
|
1636
2398
|
init_readOptions();
|
|
2399
|
+
__name(numberOf, "numberOf");
|
|
2400
|
+
__name(textOf, "textOf");
|
|
1637
2401
|
QvdColumn = class {
|
|
2402
|
+
static {
|
|
2403
|
+
__name(this, "QvdColumn");
|
|
2404
|
+
}
|
|
1638
2405
|
/**
|
|
1639
2406
|
* @param {string} name The field name.
|
|
1640
2407
|
* @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
|
|
1641
2408
|
* @param {Array<any>} symbols The field's distinct values, indexed by code.
|
|
2409
|
+
* @param {SymbolHalves|null} [halves=null] Both halves of each symbol, aligned with `symbols`, for a
|
|
2410
|
+
* field whose values do not show them all - a dual read as one half, a string read as a number.
|
|
2411
|
+
* Without them, the halves are derived from the values: a string is its own text, a dual has its
|
|
2412
|
+
* own, and a number has none.
|
|
1642
2413
|
*/
|
|
1643
|
-
constructor(name, codes, symbols) {
|
|
2414
|
+
constructor(name, codes, symbols, halves = null) {
|
|
1644
2415
|
this._name = name;
|
|
1645
2416
|
this._codes = codes;
|
|
1646
2417
|
this._symbols = symbols;
|
|
2418
|
+
this._halves = halves;
|
|
1647
2419
|
Object.freeze(this);
|
|
1648
2420
|
}
|
|
1649
2421
|
/** @return {string} The field name. */
|
|
@@ -1670,6 +2442,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1670
2442
|
*
|
|
1671
2443
|
* One entry per distinct value, not per row: a few thousand entries for a column of millions.
|
|
1672
2444
|
*
|
|
2445
|
+
* A windowed read that filters the symbol table decodes only the symbols its rows use. Every other
|
|
2446
|
+
* entry is `undefined` - not `null`, which a QVD never stores as a symbol - and no code refers to it.
|
|
2447
|
+
*
|
|
1673
2448
|
* @return {ReadonlyArray<any>} The dictionary.
|
|
1674
2449
|
*/
|
|
1675
2450
|
get symbols() {
|
|
@@ -1693,6 +2468,42 @@ var init_QvdColumnTable = __esm({
|
|
|
1693
2468
|
const code = this._codes[row];
|
|
1694
2469
|
return code < 0 ? null : this._symbols[code];
|
|
1695
2470
|
}
|
|
2471
|
+
/**
|
|
2472
|
+
* The text of one row: the text Qlik displays for its value.
|
|
2473
|
+
*
|
|
2474
|
+
* A string is its own text and a dual has its own. A value read as one half of a symbol - a date
|
|
2475
|
+
* read as its serial, a string read as a number - has the text the file stores for it, and a pure
|
|
2476
|
+
* number has none.
|
|
2477
|
+
*
|
|
2478
|
+
* @param {number} row The row index.
|
|
2479
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
2480
|
+
* @throws {QvdValidationError} If the row is not an integer within the column.
|
|
2481
|
+
*/
|
|
2482
|
+
textAt(row) {
|
|
2483
|
+
if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
|
|
2484
|
+
throw new QvdValidationError("Row index out of bounds", {
|
|
2485
|
+
column: this._name,
|
|
2486
|
+
row,
|
|
2487
|
+
length: this._codes.length
|
|
2488
|
+
});
|
|
2489
|
+
}
|
|
2490
|
+
const code = this._codes[row];
|
|
2491
|
+
if (code < 0) {
|
|
2492
|
+
return null;
|
|
2493
|
+
}
|
|
2494
|
+
return this._halves !== null ? this._halves.texts[code] ?? null : textOf(this._symbols[code]);
|
|
2495
|
+
}
|
|
2496
|
+
/**
|
|
2497
|
+
* The text of each distinct value, indexed by the codes, as `textAt` gives it per row.
|
|
2498
|
+
*
|
|
2499
|
+
* @return {ReadonlyArray<string|null>} One text per symbol, null where a symbol has none.
|
|
2500
|
+
*/
|
|
2501
|
+
symbolTexts() {
|
|
2502
|
+
if (this._halves !== null) {
|
|
2503
|
+
return this._halves.texts;
|
|
2504
|
+
}
|
|
2505
|
+
return Object.freeze(this._symbols.map(textOf));
|
|
2506
|
+
}
|
|
1696
2507
|
/**
|
|
1697
2508
|
* Iterates the column's values without materialising it.
|
|
1698
2509
|
*
|
|
@@ -1736,6 +2547,9 @@ var init_QvdColumnTable = __esm({
|
|
|
1736
2547
|
* one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
|
|
1737
2548
|
* codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
|
|
1738
2549
|
*
|
|
2550
|
+
* A dual is its number, however it was read: a `QvdDual` gives `.number`, and a date read with
|
|
2551
|
+
* `{duals: 'text'}` gives the serial the file stores for it, not NaN.
|
|
2552
|
+
*
|
|
1739
2553
|
* Scanning `codes` against this is the fastest way to read a column, because both sides are
|
|
1740
2554
|
* contiguous typed arrays and the dictionary fits in cache:
|
|
1741
2555
|
*
|
|
@@ -1757,8 +2571,7 @@ var init_QvdColumnTable = __esm({
|
|
|
1757
2571
|
numericSymbols() {
|
|
1758
2572
|
const out = new Float64Array(this._symbols.length);
|
|
1759
2573
|
for (let index = 0; index < this._symbols.length; index++) {
|
|
1760
|
-
|
|
1761
|
-
out[index] = typeof value === "number" ? value : NaN;
|
|
2574
|
+
out[index] = numberOf(this._symbols[index], this._halves?.numbers[index] ?? null);
|
|
1762
2575
|
}
|
|
1763
2576
|
return out;
|
|
1764
2577
|
}
|
|
@@ -1773,7 +2586,8 @@ var init_QvdColumnTable = __esm({
|
|
|
1773
2586
|
* @param {Object} [options] Conversion options.
|
|
1774
2587
|
* @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
|
|
1775
2588
|
* number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
|
|
1776
|
-
* NaN, which is the right choice only when the caller knows the column is numeric.
|
|
2589
|
+
* NaN, which is the right choice only when the caller knows the column is numeric. A dual is a
|
|
2590
|
+
* number here, as it is to `numericSymbols`.
|
|
1777
2591
|
* @return {Float64Array} One number per row.
|
|
1778
2592
|
* @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
|
|
1779
2593
|
*/
|
|
@@ -1793,6 +2607,11 @@ var init_QvdColumnTable = __esm({
|
|
|
1793
2607
|
out[row] = value;
|
|
1794
2608
|
continue;
|
|
1795
2609
|
}
|
|
2610
|
+
const number = value === null ? NaN : numberOf(value, this._halves?.numbers[code] ?? null);
|
|
2611
|
+
if (!Number.isNaN(number)) {
|
|
2612
|
+
out[row] = number;
|
|
2613
|
+
continue;
|
|
2614
|
+
}
|
|
1796
2615
|
if (onNonNumeric === "throw") {
|
|
1797
2616
|
throw new QvdValidationError("Column holds a value that is not a number", {
|
|
1798
2617
|
column: this._name,
|
|
@@ -1808,21 +2627,30 @@ var init_QvdColumnTable = __esm({
|
|
|
1808
2627
|
}
|
|
1809
2628
|
};
|
|
1810
2629
|
QvdColumnTable = class {
|
|
2630
|
+
static {
|
|
2631
|
+
__name(this, "QvdColumnTable");
|
|
2632
|
+
}
|
|
1811
2633
|
/**
|
|
1812
2634
|
* @param {Object} decoded What the reader decoded.
|
|
1813
2635
|
* @param {Array<string>} decoded.columns Field names, in file order.
|
|
1814
2636
|
* @param {Array<Int32Array>} decoded.codesByField One code array per field.
|
|
1815
2637
|
* @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
|
|
2638
|
+
* @param {Array<SymbolHalves|null>} [decoded.halvesByField] Both halves of each symbol, per field,
|
|
2639
|
+
* or null for a field whose values show them.
|
|
1816
2640
|
* @param {number} decoded.rowCount Rows decoded.
|
|
1817
2641
|
* @param {any} decoded.metadata The raw QvdTableHeader.
|
|
2642
|
+
* @param {import('./util/storedSymbols.js').StoredSymbols|null} [decoded.storedSymbols] The
|
|
2643
|
+
* stored-symbol record, as a data frame of the same read carries it.
|
|
1818
2644
|
* @param {any} decoded.loadStats Statistics about the read.
|
|
1819
2645
|
*/
|
|
1820
|
-
constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
|
|
2646
|
+
constructor({ columns, codesByField, symbolsByField, halvesByField, rowCount, metadata, storedSymbols, loadStats }) {
|
|
1821
2647
|
this._columns = columns;
|
|
1822
2648
|
this._codesByField = codesByField;
|
|
1823
2649
|
this._symbolsByField = symbolsByField;
|
|
2650
|
+
this._halvesByField = halvesByField ?? null;
|
|
1824
2651
|
this._rowCount = rowCount;
|
|
1825
2652
|
this._metadata = metadata;
|
|
2653
|
+
this._storedSymbols = storedSymbols ?? null;
|
|
1826
2654
|
this._loadStats = loadStats;
|
|
1827
2655
|
}
|
|
1828
2656
|
/**
|
|
@@ -1841,6 +2669,13 @@ var init_QvdColumnTable = __esm({
|
|
|
1841
2669
|
* @param {number} [options.offset] File row to start at.
|
|
1842
2670
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
1843
2671
|
* appear. Unselected fields have their symbols skipped entirely.
|
|
2672
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol's value is: its
|
|
2673
|
+
* number, its text, or a frozen `QvdDual` holding both. Whichever it is, `column.textAt` gives the
|
|
2674
|
+
* text and `numericSymbols` the number. Anything else throws.
|
|
2675
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a value that would be a string is a
|
|
2676
|
+
* number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
2677
|
+
* `Number(text)`, a dual read as text as its stored number. `column.textAt` still gives the text.
|
|
2678
|
+
* Anything but a boolean throws.
|
|
1844
2679
|
* @param {Function} [options.onProgress] Progress callback, `{stage, current, total, percent}`.
|
|
1845
2680
|
* @param {AbortSignal} [options.signal] Cancels the read.
|
|
1846
2681
|
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
@@ -1880,6 +2715,14 @@ var init_QvdColumnTable = __esm({
|
|
|
1880
2715
|
get loadStats() {
|
|
1881
2716
|
return this._loadStats;
|
|
1882
2717
|
}
|
|
2718
|
+
/**
|
|
2719
|
+
* The stored-symbol record of the read, as `QvdDataFrame.storedSymbols` describes it.
|
|
2720
|
+
*
|
|
2721
|
+
* @return {import('./util/storedSymbols.js').StoredSymbols|null} The record, or null.
|
|
2722
|
+
*/
|
|
2723
|
+
get storedSymbols() {
|
|
2724
|
+
return this._storedSymbols;
|
|
2725
|
+
}
|
|
1883
2726
|
/**
|
|
1884
2727
|
* One column.
|
|
1885
2728
|
*
|
|
@@ -1895,7 +2738,12 @@ var init_QvdColumnTable = __esm({
|
|
|
1895
2738
|
availableColumns: this._columns
|
|
1896
2739
|
});
|
|
1897
2740
|
}
|
|
1898
|
-
return new QvdColumn(
|
|
2741
|
+
return new QvdColumn(
|
|
2742
|
+
name,
|
|
2743
|
+
this._codesByField[index],
|
|
2744
|
+
this._symbolsByField[index],
|
|
2745
|
+
this._halvesByField?.[index] ?? null
|
|
2746
|
+
);
|
|
1899
2747
|
}
|
|
1900
2748
|
};
|
|
1901
2749
|
}
|
|
@@ -1906,6 +2754,16 @@ var QvdFileReader_exports = {};
|
|
|
1906
2754
|
__export(QvdFileReader_exports, {
|
|
1907
2755
|
QvdFileReader: () => QvdFileReader
|
|
1908
2756
|
});
|
|
2757
|
+
function closeReadStream(stream) {
|
|
2758
|
+
if (stream.closed) {
|
|
2759
|
+
return Promise.resolve();
|
|
2760
|
+
}
|
|
2761
|
+
return new Promise((resolve) => {
|
|
2762
|
+
stream.once("close", () => resolve());
|
|
2763
|
+
stream.once("error", () => resolve());
|
|
2764
|
+
stream.destroy();
|
|
2765
|
+
});
|
|
2766
|
+
}
|
|
1909
2767
|
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
|
|
1910
2768
|
var init_QvdFileReader = __esm({
|
|
1911
2769
|
"src/QvdFileReader.js"() {
|
|
@@ -1917,10 +2775,16 @@ var init_QvdFileReader = __esm({
|
|
|
1917
2775
|
init_validationUtils();
|
|
1918
2776
|
init_symbolParser();
|
|
1919
2777
|
init_readOptions();
|
|
2778
|
+
init_resolveSymbols();
|
|
2779
|
+
init_storedSymbols();
|
|
1920
2780
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
1921
2781
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
1922
2782
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
2783
|
+
__name(closeReadStream, "closeReadStream");
|
|
1923
2784
|
QvdFileReader = class {
|
|
2785
|
+
static {
|
|
2786
|
+
__name(this, "QvdFileReader");
|
|
2787
|
+
}
|
|
1924
2788
|
/**
|
|
1925
2789
|
* Constructs a new QVD file parser.
|
|
1926
2790
|
*
|
|
@@ -1947,6 +2811,18 @@ var init_QvdFileReader = __esm({
|
|
|
1947
2811
|
* smaller files, raise it to keep the simpler single-pass read for longer.
|
|
1948
2812
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
1949
2813
|
* appear. Null reads every field, in file order. An unknown or repeated name is refused.
|
|
2814
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
2815
|
+
* text Qlik displays for it, such as a date - reads as. `'number'` gives its number, which is the
|
|
2816
|
+
* value Qlik sums, sorts and compares by; `'text'` gives its text; `'both'` gives a frozen
|
|
2817
|
+
* `QvdDual` holding both halves, shared by every row that holds the symbol. Under `'number'` and
|
|
2818
|
+
* `'text'` the half a cell does not show is kept in the frame's `storedSymbols`, so a write stores
|
|
2819
|
+
* the dual again. An int, a double, a string and NULL read the same in every mode. Any other value
|
|
2820
|
+
* throws a `QvdValidationError`.
|
|
2821
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
2822
|
+
* reads as a number when its text is not blank and `Number(text)` is finite: a string symbol in
|
|
2823
|
+
* every `duals` mode, as `Number(text)`, and a dual's text under `duals: 'text'`, as the number the
|
|
2824
|
+
* dual stores. The text is kept in the frame's `storedSymbols`, so a write stores the string or the
|
|
2825
|
+
* dual again. Anything but a boolean, `undefined` or null throws a `QvdValidationError`.
|
|
1950
2826
|
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
1951
2827
|
* read proceeds - the same shape `QvdFileWriter` emits.
|
|
1952
2828
|
* @param {AbortSignal} [options.signal] Cancels the read. When it is aborted the read throws
|
|
@@ -1959,11 +2835,15 @@ var init_QvdFileReader = __esm({
|
|
|
1959
2835
|
symbolFilteringThreshold = 50 * 1024 * 1024,
|
|
1960
2836
|
materialisesRows = true,
|
|
1961
2837
|
fields = null,
|
|
2838
|
+
duals,
|
|
2839
|
+
coerceNumericStrings,
|
|
1962
2840
|
onProgress,
|
|
1963
2841
|
signal
|
|
1964
2842
|
} = options;
|
|
1965
2843
|
this._materialisesRows = materialisesRows;
|
|
1966
2844
|
this._path = validatePath(filePath, allowedDir);
|
|
2845
|
+
this._duals = normaliseDuals(duals, this._path);
|
|
2846
|
+
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
1967
2847
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
1968
2848
|
this._symbolFilteringThreshold = symbolFilteringThreshold;
|
|
1969
2849
|
if (onProgress !== void 0 && typeof onProgress !== "function") {
|
|
@@ -2035,15 +2915,17 @@ var init_QvdFileReader = __esm({
|
|
|
2035
2915
|
/**
|
|
2036
2916
|
* Reads the binary data of the QVD file.
|
|
2037
2917
|
*
|
|
2038
|
-
*
|
|
2039
|
-
*
|
|
2918
|
+
* A windowed read - anything with `offset`, `limit` or `maxRows` - reads only the bytes it
|
|
2919
|
+
* needs, rather than the file. Measured on `chicago_taxi_rides_2016_01.qvd`, 1,705,805 rows
|
|
2920
|
+
* over 20 fields: the last thousand rows take 19 ms against 636 ms for the whole file.
|
|
2040
2921
|
*
|
|
2041
|
-
*
|
|
2042
|
-
*
|
|
2043
|
-
*
|
|
2044
|
-
*
|
|
2922
|
+
* The saving is in the index table and the rows, not in the symbol table, which is read in
|
|
2923
|
+
* full whatever the window because a stored index in any row can address any symbol. So the
|
|
2924
|
+
* gain scales with how much of the file is rows: on a file whose bytes are mostly distinct
|
|
2925
|
+
* values there is very little to save, which is what `symbolFilteringThreshold` and the
|
|
2926
|
+
* two-pass path exist for.
|
|
2045
2927
|
*
|
|
2046
|
-
* Algorithm for
|
|
2928
|
+
* Algorithm for a windowed read:
|
|
2047
2929
|
* 1. Stream-read the file until XML header delimiter is found
|
|
2048
2930
|
* 2. Parse header to determine symbol table and index table locations
|
|
2049
2931
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
@@ -2114,6 +2996,8 @@ var init_QvdFileReader = __esm({
|
|
|
2114
2996
|
if (!isExpectedEarlyClose) {
|
|
2115
2997
|
throw error;
|
|
2116
2998
|
}
|
|
2999
|
+
} finally {
|
|
3000
|
+
await closeReadStream(stream);
|
|
2117
3001
|
}
|
|
2118
3002
|
if (headerDelimiterIndex === -1) {
|
|
2119
3003
|
throw new QvdCorruptedError(
|
|
@@ -2488,27 +3372,16 @@ var init_QvdFileReader = __esm({
|
|
|
2488
3372
|
this._throwIfAborted();
|
|
2489
3373
|
const symbolsOffset = parseInt(field["Offset"], 10);
|
|
2490
3374
|
const symbolsLength = parseInt(field["Length"], 10);
|
|
2491
|
-
const
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
symbolBuffer,
|
|
2502
|
-
pointer,
|
|
2503
|
-
symbolBuffer.length,
|
|
2504
|
-
fieldName,
|
|
2505
|
-
this._path,
|
|
2506
|
-
shouldKeepSymbol
|
|
2507
|
-
);
|
|
2508
|
-
symbols.push(symbol);
|
|
2509
|
-
pointer += bytesRead - 1;
|
|
2510
|
-
symbolIndex++;
|
|
2511
|
-
}
|
|
3375
|
+
const symbols = parseFieldSymbols(
|
|
3376
|
+
symbolBuffer,
|
|
3377
|
+
symbolsOffset,
|
|
3378
|
+
symbolsOffset + symbolsLength,
|
|
3379
|
+
// By position, matching how `_analyzeIndexTableSymbolUsage` built it. Both walk
|
|
3380
|
+
// `this._selectedFields`, so position is the one key that cannot collide.
|
|
3381
|
+
symbolsToKeep ? symbolsToKeep[position] : null,
|
|
3382
|
+
field["FieldName"],
|
|
3383
|
+
this._path
|
|
3384
|
+
);
|
|
2512
3385
|
this._emitProgress("symbol-table", position + 1, fields.length);
|
|
2513
3386
|
return symbols;
|
|
2514
3387
|
});
|
|
@@ -2619,10 +3492,16 @@ var init_QvdFileReader = __esm({
|
|
|
2619
3492
|
const prepared = await this._prepare(rows);
|
|
2620
3493
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
2621
3494
|
const data = this._buildRows(prepared.resolvedByField, 0, prepared.rowsAvailable);
|
|
2622
|
-
return new QvdDataFrame(
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
3495
|
+
return new QvdDataFrame(
|
|
3496
|
+
data,
|
|
3497
|
+
prepared.columns,
|
|
3498
|
+
prepared.metadata,
|
|
3499
|
+
{
|
|
3500
|
+
...prepared.loadStats,
|
|
3501
|
+
rowsLoaded: data.length
|
|
3502
|
+
},
|
|
3503
|
+
prepared.storedSymbols
|
|
3504
|
+
);
|
|
2626
3505
|
}
|
|
2627
3506
|
/**
|
|
2628
3507
|
* Reads the file as columns, without ever materialising rows.
|
|
@@ -2640,7 +3519,7 @@ var init_QvdFileReader = __esm({
|
|
|
2640
3519
|
*/
|
|
2641
3520
|
async loadColumnar(window = null) {
|
|
2642
3521
|
const rows = normaliseWindow(window, this._path);
|
|
2643
|
-
const prepared = await this._prepare(rows);
|
|
3522
|
+
const prepared = await this._prepare(rows, null, true);
|
|
2644
3523
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
2645
3524
|
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
2646
3525
|
assert2(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
@@ -2648,8 +3527,10 @@ var init_QvdFileReader = __esm({
|
|
|
2648
3527
|
columns: prepared.columns,
|
|
2649
3528
|
codesByField: this._indexColumns,
|
|
2650
3529
|
symbolsByField: prepared.resolvedByField,
|
|
3530
|
+
halvesByField: prepared.halvesByField,
|
|
2651
3531
|
rowCount: this._rowsDecoded,
|
|
2652
3532
|
metadata: prepared.metadata,
|
|
3533
|
+
storedSymbols: prepared.storedSymbols,
|
|
2653
3534
|
loadStats: { ...prepared.loadStats, rowsLoaded: this._rowsDecoded }
|
|
2654
3535
|
});
|
|
2655
3536
|
}
|
|
@@ -2694,11 +3575,17 @@ var init_QvdFileReader = __esm({
|
|
|
2694
3575
|
const offset = prepared.offset + done;
|
|
2695
3576
|
await this._parseIndexTable({ offset, limit: count });
|
|
2696
3577
|
const data = this._buildRows(prepared.resolvedByField, done, prepared.rowsAvailable);
|
|
2697
|
-
yield new QvdDataFrame(
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
3578
|
+
yield new QvdDataFrame(
|
|
3579
|
+
data,
|
|
3580
|
+
prepared.columns,
|
|
3581
|
+
prepared.metadata,
|
|
3582
|
+
{
|
|
3583
|
+
...prepared.loadStats,
|
|
3584
|
+
offset,
|
|
3585
|
+
rowsLoaded: data.length
|
|
3586
|
+
},
|
|
3587
|
+
prepared.storedSymbols
|
|
3588
|
+
);
|
|
2702
3589
|
}
|
|
2703
3590
|
}
|
|
2704
3591
|
/**
|
|
@@ -2715,12 +3602,17 @@ var init_QvdFileReader = __esm({
|
|
|
2715
3602
|
* @param {{rows: number, perChunk: number}|null} [liveRows] Rows held at one instant when that
|
|
2716
3603
|
* is fewer than the window covers, and how many of them one row of the caller's chunk size
|
|
2717
3604
|
* accounts for. Only `iterateRows` passes it; every other read holds what it covers.
|
|
3605
|
+
* @param {boolean} [wantHalves=false] Whether to keep both halves of each symbol of a field whose
|
|
3606
|
+
* cells do not show them, which only a columnar read has a use for.
|
|
2718
3607
|
* @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
|
|
2719
|
-
* resolvedByField: Array<Array<any>>,
|
|
2720
|
-
*
|
|
3608
|
+
* resolvedByField: Array<Array<any>>,
|
|
3609
|
+
* halvesByField: Array<import('./util/resolveSymbols.js').SymbolHalves|null>,
|
|
3610
|
+
* storedSymbols: import('./util/storedSymbols.js').StoredSymbols|null,
|
|
3611
|
+
* rowsAvailable: number, offset: number}>} The parsed file, with the window as it resolved
|
|
3612
|
+
* against it.
|
|
2721
3613
|
* @private
|
|
2722
3614
|
*/
|
|
2723
|
-
async _prepare(window, liveRows = null) {
|
|
3615
|
+
async _prepare(window, liveRows = null, wantHalves = false) {
|
|
2724
3616
|
this._throwIfAborted();
|
|
2725
3617
|
await this._readData(window, false, liveRows);
|
|
2726
3618
|
this._emitProgress("header", 0, 1);
|
|
@@ -2743,17 +3635,31 @@ var init_QvdFileReader = __esm({
|
|
|
2743
3635
|
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
2744
3636
|
assert2(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
2745
3637
|
this._throwIfAborted();
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
3638
|
+
assert2(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3639
|
+
const resolvedByField = [];
|
|
3640
|
+
const halvesByField = [];
|
|
3641
|
+
const entries = [];
|
|
3642
|
+
this._symbolTable.forEach((symbols, position) => {
|
|
3643
|
+
const { values, entry, halves } = resolveFieldSymbols(
|
|
3644
|
+
symbols,
|
|
3645
|
+
// @ts-ignore - asserted above
|
|
3646
|
+
this._selectedFields[position]["FieldName"],
|
|
3647
|
+
this._duals,
|
|
3648
|
+
this._coerceNumericStrings,
|
|
3649
|
+
wantHalves
|
|
3650
|
+
);
|
|
3651
|
+
resolvedByField.push(values);
|
|
3652
|
+
halvesByField.push(halves);
|
|
3653
|
+
if (entry !== null) {
|
|
3654
|
+
entries.push(entry);
|
|
2751
3655
|
}
|
|
2752
|
-
return resolved2;
|
|
2753
3656
|
});
|
|
2754
|
-
assert2(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
2755
3657
|
const columns = this._selectedFields.map((field) => field["FieldName"]);
|
|
2756
3658
|
const metadata = this._header["QvdTableHeader"];
|
|
3659
|
+
const storedSymbols = entries.length > 0 ? trustStoredSymbols(entries) : null;
|
|
3660
|
+
if (storedSymbols !== null) {
|
|
3661
|
+
attachStoredSymbols(metadata, storedSymbols);
|
|
3662
|
+
}
|
|
2757
3663
|
const loadStats = {
|
|
2758
3664
|
symbolTableBytes: symbolTableLength,
|
|
2759
3665
|
totalRows,
|
|
@@ -2762,7 +3668,16 @@ var init_QvdFileReader = __esm({
|
|
|
2762
3668
|
symbolFiltering: symbolsToKeep !== null,
|
|
2763
3669
|
symbolsKept
|
|
2764
3670
|
};
|
|
2765
|
-
return {
|
|
3671
|
+
return {
|
|
3672
|
+
columns,
|
|
3673
|
+
metadata,
|
|
3674
|
+
loadStats,
|
|
3675
|
+
resolvedByField,
|
|
3676
|
+
halvesByField,
|
|
3677
|
+
storedSymbols,
|
|
3678
|
+
rowsAvailable,
|
|
3679
|
+
offset: resolved.offset
|
|
3680
|
+
};
|
|
2766
3681
|
}
|
|
2767
3682
|
/**
|
|
2768
3683
|
* Builds rows from the columns currently decoded.
|
|
@@ -2806,24 +3721,95 @@ var init_QvdFileReader = __esm({
|
|
|
2806
3721
|
});
|
|
2807
3722
|
|
|
2808
3723
|
// src/QvdDataFrame.js
|
|
3724
|
+
function defaultFieldHeader(fieldName) {
|
|
3725
|
+
return {
|
|
3726
|
+
FieldName: fieldName,
|
|
3727
|
+
BitOffset: 0,
|
|
3728
|
+
BitWidth: 0,
|
|
3729
|
+
Bias: 0,
|
|
3730
|
+
NoOfSymbols: 0,
|
|
3731
|
+
Offset: 0,
|
|
3732
|
+
Length: 0,
|
|
3733
|
+
Comment: "",
|
|
3734
|
+
NumberFormat: {
|
|
3735
|
+
Type: "UNKNOWN",
|
|
3736
|
+
nDec: "0",
|
|
3737
|
+
UseThou: "0",
|
|
3738
|
+
Fmt: "",
|
|
3739
|
+
Dec: "",
|
|
3740
|
+
Thou: ""
|
|
3741
|
+
},
|
|
3742
|
+
Tags: {}
|
|
3743
|
+
};
|
|
3744
|
+
}
|
|
3745
|
+
function defaultHeader(columns) {
|
|
3746
|
+
return {
|
|
3747
|
+
QvBuildNo: 50667,
|
|
3748
|
+
CreatorDoc: "",
|
|
3749
|
+
CreateUtcTime: "",
|
|
3750
|
+
SourceCreateUtcTime: "",
|
|
3751
|
+
SourceFileUtcTime: "",
|
|
3752
|
+
SourceFileSize: -1,
|
|
3753
|
+
StaleUtcTime: "",
|
|
3754
|
+
TableName: "",
|
|
3755
|
+
Fields: {
|
|
3756
|
+
QvdFieldHeader: columns.map(defaultFieldHeader)
|
|
3757
|
+
},
|
|
3758
|
+
NoOfRecords: 0,
|
|
3759
|
+
RecordByteSize: 0,
|
|
3760
|
+
Offset: 0,
|
|
3761
|
+
Length: 0,
|
|
3762
|
+
Compression: "",
|
|
3763
|
+
Comment: "",
|
|
3764
|
+
EncryptionInfo: "",
|
|
3765
|
+
TableTags: "",
|
|
3766
|
+
ProfilingData: "",
|
|
3767
|
+
Lineage: {}
|
|
3768
|
+
};
|
|
3769
|
+
}
|
|
2809
3770
|
var QvdDataFrame;
|
|
2810
3771
|
var init_QvdDataFrame = __esm({
|
|
2811
3772
|
"src/QvdDataFrame.js"() {
|
|
2812
3773
|
init_QvdErrors();
|
|
3774
|
+
init_cellRules();
|
|
2813
3775
|
init_readOptions();
|
|
3776
|
+
init_storedSymbols();
|
|
3777
|
+
__name(defaultFieldHeader, "defaultFieldHeader");
|
|
3778
|
+
__name(defaultHeader, "defaultHeader");
|
|
2814
3779
|
QvdDataFrame = class _QvdDataFrame {
|
|
3780
|
+
static {
|
|
3781
|
+
__name(this, "QvdDataFrame");
|
|
3782
|
+
}
|
|
2815
3783
|
/**
|
|
2816
3784
|
* Represents the data frame stored inside a QVD file.
|
|
3785
|
+
*
|
|
3786
|
+
* The record is resolved once, here: the fifth argument when given, otherwise the one a read left on
|
|
3787
|
+
* its header object, so `new QvdDataFrame(data, columns, df.metadata)` keeps what `df` would write.
|
|
3788
|
+
* Either is narrowed to `columns`. An entry for a field the frame does not have describes no cell it
|
|
3789
|
+
* holds, and would make the frame's own `toDict()` a dictionary `fromDict` refuses - which is what a
|
|
3790
|
+
* header's record did for a frame built from some of a read's columns.
|
|
3791
|
+
*
|
|
2817
3792
|
* @param {Array<Array<any>>} data The data of the data frame.
|
|
2818
3793
|
* @param {Array<string>} columns The columns of the data frame.
|
|
2819
3794
|
* @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
|
|
2820
3795
|
* @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
|
|
3796
|
+
* @param {QvdStoredSymbols|null} storedSymbols What the frame's cells were read from, where a cell
|
|
3797
|
+
* shows only one half of its symbol (optional) - see `storedSymbols`.
|
|
3798
|
+
* @throws {QvdValidationError} If the record is malformed.
|
|
2821
3799
|
*/
|
|
2822
|
-
constructor(data, columns, metadata = null, loadStats = null) {
|
|
3800
|
+
constructor(data, columns, metadata = null, loadStats = null, storedSymbols = null) {
|
|
2823
3801
|
this._data = data;
|
|
2824
3802
|
this._columns = columns;
|
|
2825
3803
|
this._metadata = metadata;
|
|
3804
|
+
this._ownsMetadata = false;
|
|
2826
3805
|
this._loadStats = loadStats;
|
|
3806
|
+
this._storedSymbols = narrowStoredSymbols(
|
|
3807
|
+
normaliseStoredSymbols(
|
|
3808
|
+
// @ts-ignore - a symbol-keyed property the reader defines on the header object
|
|
3809
|
+
storedSymbols ?? (metadata !== null && typeof metadata === "object" ? metadata[STORED_SYMBOLS] : null)
|
|
3810
|
+
),
|
|
3811
|
+
columns
|
|
3812
|
+
);
|
|
2827
3813
|
}
|
|
2828
3814
|
/**
|
|
2829
3815
|
* Returns the data of the data frame.
|
|
@@ -2850,6 +3836,28 @@ var init_QvdDataFrame = __esm({
|
|
|
2850
3836
|
get metadata() {
|
|
2851
3837
|
return this._metadata;
|
|
2852
3838
|
}
|
|
3839
|
+
/**
|
|
3840
|
+
* What the frame's cells were read from, where a cell shows only one half of its symbol.
|
|
3841
|
+
*
|
|
3842
|
+
* A dual read as its number has a text the cell does not show; one read as its text has a number;
|
|
3843
|
+
* a string read as a number has the text it was spelled with. The record keeps those halves, per
|
|
3844
|
+
* field, keyed by the value the cell holds, so `toQvd` writes the symbols the frame was read from
|
|
3845
|
+
* and `textAt` can return any cell's text. It moves with the frame through `head`, `tail`, `rows`,
|
|
3846
|
+
* `select`, `toDict` and `fromDict`.
|
|
3847
|
+
*
|
|
3848
|
+
* Frozen plain data: `[{field, values, numbers, texts}]`, where a cell holding `values[i]` stands for
|
|
3849
|
+
* the stored symbol (`numbers[i]`, `texts[i]`), a null number meaning a pure string and a null text a
|
|
3850
|
+
* pure number.
|
|
3851
|
+
*
|
|
3852
|
+
* @return {QvdStoredSymbols|null} The record, or null when the frame has none: every cell of the read
|
|
3853
|
+
* showed its whole symbol, or the frame was built without one. A frame whose columns have no entry
|
|
3854
|
+
* in the record it was given or found - one from `select`, or one built from some of a read's
|
|
3855
|
+
* columns and its header - has an empty record rather than null, because a frame given null takes
|
|
3856
|
+
* the record its header carries, which describes every field of the read.
|
|
3857
|
+
*/
|
|
3858
|
+
get storedSymbols() {
|
|
3859
|
+
return this._storedSymbols;
|
|
3860
|
+
}
|
|
2853
3861
|
/**
|
|
2854
3862
|
* Returns statistics about the read that produced this data frame.
|
|
2855
3863
|
*
|
|
@@ -2970,54 +3978,40 @@ var init_QvdDataFrame = __esm({
|
|
|
2970
3978
|
* @property {string} [profilingData] - Profiling data
|
|
2971
3979
|
* @property {Object|string} [lineage] - Lineage
|
|
2972
3980
|
*/
|
|
3981
|
+
/**
|
|
3982
|
+
* The header a metadata setter may change: this frame's own.
|
|
3983
|
+
*
|
|
3984
|
+
* A frame's header can be shared. `head`, `tail`, `rows` and `select` pass theirs on, every chunk
|
|
3985
|
+
* `iterate()` yields holds the same one, and `fromDict` uses the object it is given. So the first
|
|
3986
|
+
* change copies it, and a change made through one frame never reaches another. A frame with no header
|
|
3987
|
+
* gets the one `toQvd` would write for it. The copy keeps the stored-symbol record the header carries,
|
|
3988
|
+
* so `new QvdDataFrame(data, columns, df.metadata)` still writes what `df` would.
|
|
3989
|
+
*
|
|
3990
|
+
* @return {any} The header.
|
|
3991
|
+
*/
|
|
3992
|
+
_ownMetadata() {
|
|
3993
|
+
if (!this._metadata) {
|
|
3994
|
+
this._metadata = defaultHeader(this._columns);
|
|
3995
|
+
} else if (!this._ownsMetadata) {
|
|
3996
|
+
const record = this._metadata[STORED_SYMBOLS];
|
|
3997
|
+
this._metadata = structuredClone(this._metadata);
|
|
3998
|
+
if (record) {
|
|
3999
|
+
attachStoredSymbols(this._metadata, record);
|
|
4000
|
+
}
|
|
4001
|
+
}
|
|
4002
|
+
this._ownsMetadata = true;
|
|
4003
|
+
return this._metadata;
|
|
4004
|
+
}
|
|
2973
4005
|
/**
|
|
2974
4006
|
* Sets modifiable file-level metadata. Immutable properties related to data storage are ignored.
|
|
4007
|
+
*
|
|
4008
|
+
* The change applies to this frame only, never to a frame it was derived from or shares a header
|
|
4009
|
+
* with.
|
|
4010
|
+
*
|
|
2975
4011
|
* @param {FileMetadataUpdate} metadata Object containing metadata properties to update.
|
|
2976
4012
|
*/
|
|
2977
4013
|
setFileMetadata(metadata) {
|
|
2978
|
-
|
|
2979
|
-
this._metadata = {
|
|
2980
|
-
QvBuildNo: 50667,
|
|
2981
|
-
CreatorDoc: "",
|
|
2982
|
-
CreateUtcTime: "",
|
|
2983
|
-
SourceCreateUtcTime: "",
|
|
2984
|
-
SourceFileUtcTime: "",
|
|
2985
|
-
SourceFileSize: -1,
|
|
2986
|
-
StaleUtcTime: "",
|
|
2987
|
-
TableName: "",
|
|
2988
|
-
Fields: {
|
|
2989
|
-
QvdFieldHeader: this._columns.map((column) => ({
|
|
2990
|
-
FieldName: column,
|
|
2991
|
-
BitOffset: 0,
|
|
2992
|
-
BitWidth: 0,
|
|
2993
|
-
Bias: 0,
|
|
2994
|
-
NoOfSymbols: 0,
|
|
2995
|
-
Offset: 0,
|
|
2996
|
-
Length: 0,
|
|
2997
|
-
Comment: "",
|
|
2998
|
-
NumberFormat: {
|
|
2999
|
-
Type: "UNKNOWN",
|
|
3000
|
-
nDec: "0",
|
|
3001
|
-
UseThou: "0",
|
|
3002
|
-
Fmt: "",
|
|
3003
|
-
Dec: "",
|
|
3004
|
-
Thou: ""
|
|
3005
|
-
},
|
|
3006
|
-
Tags: {}
|
|
3007
|
-
}))
|
|
3008
|
-
},
|
|
3009
|
-
NoOfRecords: 0,
|
|
3010
|
-
RecordByteSize: 0,
|
|
3011
|
-
Offset: 0,
|
|
3012
|
-
Length: 0,
|
|
3013
|
-
Compression: "",
|
|
3014
|
-
Comment: "",
|
|
3015
|
-
EncryptionInfo: "",
|
|
3016
|
-
TableTags: "",
|
|
3017
|
-
ProfilingData: "",
|
|
3018
|
-
Lineage: {}
|
|
3019
|
-
};
|
|
3020
|
-
}
|
|
4014
|
+
const header = this._ownMetadata();
|
|
3021
4015
|
const modifiableFields = [
|
|
3022
4016
|
"qvBuildNo",
|
|
3023
4017
|
"creatorDoc",
|
|
@@ -3052,7 +4046,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3052
4046
|
};
|
|
3053
4047
|
modifiableFields.forEach((field) => {
|
|
3054
4048
|
if (metadata[field] !== void 0) {
|
|
3055
|
-
|
|
4049
|
+
header[fieldMapping[field]] = metadata[field];
|
|
3056
4050
|
}
|
|
3057
4051
|
});
|
|
3058
4052
|
}
|
|
@@ -3065,30 +4059,44 @@ var init_QvdDataFrame = __esm({
|
|
|
3065
4059
|
/**
|
|
3066
4060
|
* Sets modifiable field-level metadata for a specific field.
|
|
3067
4061
|
* Immutable properties related to data storage (Offset, Length, BitOffset, etc.) are ignored.
|
|
4062
|
+
*
|
|
4063
|
+
* Works on any frame, including one with no header yet - one from `fromDict` - and on a column the
|
|
4064
|
+
* header does not describe. The change applies to this frame only, never to a frame it was derived
|
|
4065
|
+
* from or shares a header with.
|
|
4066
|
+
*
|
|
3068
4067
|
* @param {string} fieldName The name of the field.
|
|
3069
4068
|
* @param {FieldMetadataUpdate} metadata Object containing field metadata properties to update.
|
|
4069
|
+
* @throws {QvdValidationError} If the frame has no column of that name.
|
|
3070
4070
|
*/
|
|
3071
4071
|
setFieldMetadata(fieldName, metadata) {
|
|
3072
|
-
if (!this.
|
|
3073
|
-
|
|
4072
|
+
if (!this._columns.includes(fieldName)) {
|
|
4073
|
+
throw new QvdValidationError(`Column '${fieldName}' does not exist`, {
|
|
4074
|
+
column: fieldName,
|
|
4075
|
+
availableColumns: this._columns
|
|
4076
|
+
});
|
|
3074
4077
|
}
|
|
3075
|
-
|
|
4078
|
+
const header = this._ownMetadata();
|
|
4079
|
+
if (!header.Fields || typeof header.Fields !== "object" || !header.Fields.QvdFieldHeader) {
|
|
4080
|
+
header.Fields = { QvdFieldHeader: [] };
|
|
4081
|
+
}
|
|
4082
|
+
let fields = header.Fields.QvdFieldHeader;
|
|
3076
4083
|
if (!Array.isArray(fields)) {
|
|
3077
4084
|
fields = [fields];
|
|
3078
|
-
|
|
4085
|
+
header.Fields.QvdFieldHeader = fields;
|
|
3079
4086
|
}
|
|
3080
|
-
|
|
3081
|
-
if (
|
|
3082
|
-
|
|
4087
|
+
let field = fields.find((f) => f.FieldName === fieldName);
|
|
4088
|
+
if (!field) {
|
|
4089
|
+
field = defaultFieldHeader(fieldName);
|
|
4090
|
+
fields.push(field);
|
|
3083
4091
|
}
|
|
3084
4092
|
if (metadata.comment !== void 0) {
|
|
3085
|
-
|
|
4093
|
+
field.Comment = metadata.comment;
|
|
3086
4094
|
}
|
|
3087
4095
|
if (metadata.numberFormat !== void 0) {
|
|
3088
|
-
|
|
4096
|
+
field.NumberFormat = metadata.numberFormat;
|
|
3089
4097
|
}
|
|
3090
4098
|
if (metadata.tags !== void 0) {
|
|
3091
|
-
|
|
4099
|
+
field.Tags = metadata.tags;
|
|
3092
4100
|
}
|
|
3093
4101
|
}
|
|
3094
4102
|
/**
|
|
@@ -3105,7 +4113,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3105
4113
|
type: typeof n
|
|
3106
4114
|
});
|
|
3107
4115
|
}
|
|
3108
|
-
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata);
|
|
4116
|
+
return new _QvdDataFrame(this._data.slice(0, n), this._columns, this._metadata, null, this._storedSymbols);
|
|
3109
4117
|
}
|
|
3110
4118
|
/**
|
|
3111
4119
|
* Returns the last n rows of the data frame.
|
|
@@ -3121,7 +4129,13 @@ var init_QvdDataFrame = __esm({
|
|
|
3121
4129
|
type: typeof n
|
|
3122
4130
|
});
|
|
3123
4131
|
}
|
|
3124
|
-
return new _QvdDataFrame(
|
|
4132
|
+
return new _QvdDataFrame(
|
|
4133
|
+
n === 0 ? [] : this._data.slice(-n),
|
|
4134
|
+
this._columns,
|
|
4135
|
+
this._metadata,
|
|
4136
|
+
null,
|
|
4137
|
+
this._storedSymbols
|
|
4138
|
+
);
|
|
3125
4139
|
}
|
|
3126
4140
|
/**
|
|
3127
4141
|
* Returns the selected rows of the data frame.
|
|
@@ -3149,7 +4163,9 @@ var init_QvdDataFrame = __esm({
|
|
|
3149
4163
|
return new _QvdDataFrame(
|
|
3150
4164
|
args.map((index) => this._data[index]),
|
|
3151
4165
|
this._columns,
|
|
3152
|
-
this._metadata
|
|
4166
|
+
this._metadata,
|
|
4167
|
+
null,
|
|
4168
|
+
this._storedSymbols
|
|
3153
4169
|
);
|
|
3154
4170
|
}
|
|
3155
4171
|
/**
|
|
@@ -3161,6 +4177,50 @@ var init_QvdDataFrame = __esm({
|
|
|
3161
4177
|
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
3162
4178
|
*/
|
|
3163
4179
|
at(row, column) {
|
|
4180
|
+
const index = this._cellIndex(row, column);
|
|
4181
|
+
return this._data[row][index];
|
|
4182
|
+
}
|
|
4183
|
+
/**
|
|
4184
|
+
* Returns the text of the value at the specified row and column.
|
|
4185
|
+
*
|
|
4186
|
+
* The text Qlik displays for it: a string cell is its own text, and a dual cell's text is its
|
|
4187
|
+
* `.text`. A number cell's text comes from the frame's `storedSymbols` - the dual it was read from,
|
|
4188
|
+
* or the string it was spelled as - and is null for a number that was stored as a pure number.
|
|
4189
|
+
*
|
|
4190
|
+
* ```js
|
|
4191
|
+
* const df = await QvdDataFrame.fromQvd('stockholm_temp.qvd');
|
|
4192
|
+
* df.at(0, 'date'); // -52593
|
|
4193
|
+
* df.textAt(0, 'date'); // '1756-01-01'
|
|
4194
|
+
* ```
|
|
4195
|
+
*
|
|
4196
|
+
* @param {number} row The index of the row.
|
|
4197
|
+
* @param {string} column The name of the column.
|
|
4198
|
+
* @return {string|null} The text, or null for NULL and for a number with no text.
|
|
4199
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4200
|
+
*/
|
|
4201
|
+
textAt(row, column) {
|
|
4202
|
+
const index = this._cellIndex(row, column);
|
|
4203
|
+
const value = this._data[row][index];
|
|
4204
|
+
if (typeof value === "string") {
|
|
4205
|
+
return value;
|
|
4206
|
+
}
|
|
4207
|
+
if (typeof value === "number") {
|
|
4208
|
+
const entry = storedSymbolsEntry(this._storedSymbols, column);
|
|
4209
|
+
return entry === null ? null : storedTextOf(entry, value);
|
|
4210
|
+
}
|
|
4211
|
+
const dual = asDual(value);
|
|
4212
|
+
return dual !== null && typeof dual.text === "string" ? dual.text : null;
|
|
4213
|
+
}
|
|
4214
|
+
/**
|
|
4215
|
+
* Checks a row and a column name, and returns the column's position.
|
|
4216
|
+
*
|
|
4217
|
+
* @param {number} row The index of the row.
|
|
4218
|
+
* @param {string} column The name of the column.
|
|
4219
|
+
* @return {number} The column's position.
|
|
4220
|
+
* @throws {QvdValidationError} If row is not an integer, out of bounds, or column does not exist.
|
|
4221
|
+
* @private
|
|
4222
|
+
*/
|
|
4223
|
+
_cellIndex(row, column) {
|
|
3164
4224
|
if (typeof row !== "number" || !Number.isInteger(row)) {
|
|
3165
4225
|
throw new QvdValidationError("Row index must be an integer", {
|
|
3166
4226
|
provided: row,
|
|
@@ -3180,7 +4240,7 @@ var init_QvdDataFrame = __esm({
|
|
|
3180
4240
|
availableColumns: this._columns
|
|
3181
4241
|
});
|
|
3182
4242
|
}
|
|
3183
|
-
return this.
|
|
4243
|
+
return this._columns.indexOf(column);
|
|
3184
4244
|
}
|
|
3185
4245
|
/**
|
|
3186
4246
|
* Selects the specified columns from the data frame.
|
|
@@ -3201,15 +4261,34 @@ var init_QvdDataFrame = __esm({
|
|
|
3201
4261
|
const indices = args.map((arg) => this._columns.indexOf(arg));
|
|
3202
4262
|
const data = this._data.map((row) => indices.map((index) => row[index]));
|
|
3203
4263
|
const columns = indices.map((index) => this._columns[index]);
|
|
3204
|
-
return new _QvdDataFrame(data, columns, this._metadata);
|
|
4264
|
+
return new _QvdDataFrame(data, columns, this._metadata, null, this._storedSymbols);
|
|
3205
4265
|
}
|
|
3206
4266
|
/**
|
|
3207
4267
|
* Returns the data frame as a dictionary.
|
|
3208
4268
|
*
|
|
3209
|
-
*
|
|
4269
|
+
* Everything a frame needs to write the same file again, as plain data: the header, and the
|
|
4270
|
+
* stored-symbol record that says what cells showing one half of a symbol were read from. So
|
|
4271
|
+
* `fromDict(await df.toDict())` writes what `df` writes, and so does a dictionary that went through
|
|
4272
|
+
* `JSON`, `structuredClone` or a worker on the way. The arrays are the frame's own, not copies.
|
|
4273
|
+
*
|
|
4274
|
+
* @return {Promise<QvdDataFrameDict>} The data frame as a dictionary.
|
|
3210
4275
|
*/
|
|
3211
4276
|
async toDict() {
|
|
3212
|
-
return
|
|
4277
|
+
return this.toJSON();
|
|
4278
|
+
}
|
|
4279
|
+
/**
|
|
4280
|
+
* The same dictionary `toDict` returns, synchronously, so `JSON.stringify(df)` gives something
|
|
4281
|
+
* `fromDict(JSON.parse(...))` can revive.
|
|
4282
|
+
*
|
|
4283
|
+
* @return {QvdDataFrameDict} The data frame as a dictionary.
|
|
4284
|
+
*/
|
|
4285
|
+
toJSON() {
|
|
4286
|
+
return {
|
|
4287
|
+
columns: this._columns,
|
|
4288
|
+
data: this._data,
|
|
4289
|
+
metadata: this._metadata,
|
|
4290
|
+
storedSymbols: this._storedSymbols
|
|
4291
|
+
};
|
|
3213
4292
|
}
|
|
3214
4293
|
/**
|
|
3215
4294
|
* Persists the data frame to a QVD file.
|
|
@@ -3246,6 +4325,21 @@ var init_QvdDataFrame = __esm({
|
|
|
3246
4325
|
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should
|
|
3247
4326
|
* appear in the result. Unselected fields have their symbols skipped entirely rather than
|
|
3248
4327
|
* parsed and discarded. An unknown or repeated name throws.
|
|
4328
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol - a number with the
|
|
4329
|
+
* text Qlik displays for it, such as a date, a timestamp or a formatted amount - reads as.
|
|
4330
|
+
* `'number'` gives its number, the value Qlik sums, sorts and compares by, so a date is its serial.
|
|
4331
|
+
* `'text'` gives its text. `'both'` gives a frozen `QvdDual` with `.number` and `.text`, whose
|
|
4332
|
+
* implicit conversions throw. Under `'number'` and `'text'` the other half is kept in
|
|
4333
|
+
* `storedSymbols`, so `toQvd` writes the dual back, and `textAt` returns any cell's text. An int, a
|
|
4334
|
+
* double, a string and NULL read the same in every mode: a number, a string and null. Anything else
|
|
4335
|
+
* throws.
|
|
4336
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4337
|
+
* reads as a number when its text is not blank and `Number(text)` is finite. A string symbol then
|
|
4338
|
+
* reads as `Number(text)`, so `'007'` is 7, in every `duals` mode; a dual read with
|
|
4339
|
+
* `duals: 'text'` reads as the number it stores. Blank text, and text such as `'8E5597'` whose
|
|
4340
|
+
* `Number()` is Infinity, stay strings. The text is kept in `storedSymbols`, so `toQvd` writes the
|
|
4341
|
+
* original string or dual back, and a value that two stored values read as is refused there
|
|
4342
|
+
* rather than written as either. Anything but a boolean throws.
|
|
3249
4343
|
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}` as the
|
|
3250
4344
|
* read proceeds - the same shape `toQvd`'s callback receives.
|
|
3251
4345
|
* @param {AbortSignal} [options.signal] Cancels the read. The rejection is `signal.reason`,
|
|
@@ -3263,7 +4357,8 @@ var init_QvdDataFrame = __esm({
|
|
|
3263
4357
|
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
|
|
3264
4358
|
* a lazy load switches to the two-pass filtering path. Defaults to 50MB.
|
|
3265
4359
|
* @throws {QvdValidationError} If a window option is not a non-negative integer, if both
|
|
3266
|
-
* `maxRows` and `limit` are given,
|
|
4360
|
+
* `maxRows` and `limit` are given, if `fields` names a column the file does not have, if `duals`
|
|
4361
|
+
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
3267
4362
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
3268
4363
|
*/
|
|
3269
4364
|
static async fromQvd(path3, options = {}) {
|
|
@@ -3289,9 +4384,30 @@ var init_QvdDataFrame = __esm({
|
|
|
3289
4384
|
* A window covering no rows yields nothing, so a loop over an exhausted offset simply does not
|
|
3290
4385
|
* run its body.
|
|
3291
4386
|
*
|
|
4387
|
+
* Every chunk carries the same `storedSymbols`, because the symbol table is parsed once for all of
|
|
4388
|
+
* them, so a chunk written on its own writes the symbols its cells were read from.
|
|
4389
|
+
*
|
|
3292
4390
|
* @param {string} path The path to the QVD file.
|
|
3293
|
-
* @param {Object} [options]
|
|
4391
|
+
* @param {Object} [options] Reading options, with the meanings they have on `fromQvd`.
|
|
3294
4392
|
* @param {number} [options.chunkSize=100000] Rows per frame. Must be a positive integer.
|
|
4393
|
+
* @param {number|null} [options.maxRows] Rows to cover. The older name for `limit`.
|
|
4394
|
+
* @param {number|null} [options.limit] Rows to cover, counting from `offset`.
|
|
4395
|
+
* @param {number} [options.offset=0] File row to start at.
|
|
4396
|
+
* @param {Array<string>|null} [options.fields] Field names to read, in the order they should appear.
|
|
4397
|
+
* @param {'number'|'text'|'both'} [options.duals='number'] What a dual symbol reads as: its number,
|
|
4398
|
+
* its text, or a frozen `QvdDual` holding both. Anything else throws.
|
|
4399
|
+
* @param {boolean} [options.coerceNumericStrings=false] Whether a cell that would read as a string
|
|
4400
|
+
* reads as a number when its text is not blank and `Number(text)` is finite - a string symbol as
|
|
4401
|
+
* `Number(text)`, a dual read as text as its stored number - with the text kept in every chunk's
|
|
4402
|
+
* `storedSymbols`. Anything but a boolean throws.
|
|
4403
|
+
* @param {Function} [options.onProgress] Called with `{stage, current, total, percent}`; progress
|
|
4404
|
+
* over the rows counts the whole window, not each chunk.
|
|
4405
|
+
* @param {AbortSignal} [options.signal] Cancels the iteration, rejecting with `signal.reason`.
|
|
4406
|
+
* @param {string} [options.allowedDir] Directory the path must resolve inside.
|
|
4407
|
+
* @param {number} [options.memorySafetyFactor=0.8] Fraction of the memory budget the read may use,
|
|
4408
|
+
* charged for two chunks of rows rather than the window. Zero disables the check.
|
|
4409
|
+
* @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size above which a
|
|
4410
|
+
* windowed read switches to two-pass filtering.
|
|
3295
4411
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
3296
4412
|
*/
|
|
3297
4413
|
static async *iterate(path3, options = {}) {
|
|
@@ -3334,8 +4450,15 @@ var init_QvdDataFrame = __esm({
|
|
|
3334
4450
|
/**
|
|
3335
4451
|
* Constructs a data frame from a dictionary.
|
|
3336
4452
|
*
|
|
3337
|
-
*
|
|
4453
|
+
* Takes what `toDict` returns. `metadata` and `storedSymbols` are optional; with the record, a frame
|
|
4454
|
+
* rebuilt from a read writes the symbols the read found - duals with their texts, strings with their
|
|
4455
|
+
* spelling - even after a trip through `JSON`. The record is checked, so a malformed one is refused
|
|
4456
|
+
* here rather than written.
|
|
4457
|
+
*
|
|
4458
|
+
* @param {QvdDataFrameDict} data The dictionary to construct the data frame from.
|
|
3338
4459
|
* @return {Promise<QvdDataFrame>} The constructed data frame.
|
|
4460
|
+
* @throws {QvdValidationError} If `columns` or `data` is missing, `metadata` is not a plain object,
|
|
4461
|
+
* or `storedSymbols` is malformed or names a field that is not one of the columns.
|
|
3339
4462
|
*/
|
|
3340
4463
|
static async fromDict(data) {
|
|
3341
4464
|
if (!data.columns) {
|
|
@@ -3348,20 +4471,290 @@ var init_QvdDataFrame = __esm({
|
|
|
3348
4471
|
data
|
|
3349
4472
|
});
|
|
3350
4473
|
}
|
|
3351
|
-
|
|
4474
|
+
const { metadata = null, storedSymbols = null } = data;
|
|
4475
|
+
if (metadata !== null && !isPlainObject(metadata)) {
|
|
4476
|
+
throw new QvdValidationError(`metadata must be a plain object; got ${describeType(metadata)}`, {
|
|
4477
|
+
type: typeof metadata
|
|
4478
|
+
});
|
|
4479
|
+
}
|
|
4480
|
+
const record = normaliseStoredSymbols(storedSymbols);
|
|
4481
|
+
if (record !== null) {
|
|
4482
|
+
const unknown = record.find((entry) => !data.columns.includes(entry.field));
|
|
4483
|
+
if (unknown !== void 0) {
|
|
4484
|
+
throw new QvdValidationError(`storedSymbols names field '${unknown.field}', which is not one of the columns`, {
|
|
4485
|
+
field: unknown.field,
|
|
4486
|
+
availableColumns: data.columns
|
|
4487
|
+
});
|
|
4488
|
+
}
|
|
4489
|
+
}
|
|
4490
|
+
return new _QvdDataFrame(data.data, data.columns, metadata, null, record);
|
|
3352
4491
|
}
|
|
3353
4492
|
};
|
|
3354
4493
|
}
|
|
3355
4494
|
});
|
|
3356
4495
|
|
|
4496
|
+
// src/QvdSymbol.js
|
|
4497
|
+
init_QvdErrors();
|
|
4498
|
+
init_cellRules();
|
|
4499
|
+
init_symbolBytes();
|
|
4500
|
+
function checkInteger(value, context) {
|
|
4501
|
+
if (typeof value === "number" && isStoredAsInt(value)) {
|
|
4502
|
+
return;
|
|
4503
|
+
}
|
|
4504
|
+
throw new QvdValidationError(
|
|
4505
|
+
`The integer of a symbol must be an integer from ${INT32_MIN} to ${INT32_MAX}; got ` + (typeof value === "number" ? String(value) : describeType(value)),
|
|
4506
|
+
{ ...context, half: "integer", type: typeof value }
|
|
4507
|
+
);
|
|
4508
|
+
}
|
|
4509
|
+
__name(checkInteger, "checkInteger");
|
|
4510
|
+
var QvdSymbol = class _QvdSymbol {
|
|
4511
|
+
static {
|
|
4512
|
+
__name(this, "QvdSymbol");
|
|
4513
|
+
}
|
|
4514
|
+
/**
|
|
4515
|
+
* Constructs a new QVD symbol.
|
|
4516
|
+
*
|
|
4517
|
+
* @param {number|null} intValue The integer value.
|
|
4518
|
+
* @param {number|null} doubleValue The double value.
|
|
4519
|
+
* @param {string|null} stringValue The string value.
|
|
4520
|
+
*/
|
|
4521
|
+
constructor(intValue, doubleValue, stringValue) {
|
|
4522
|
+
this._intValue = intValue;
|
|
4523
|
+
this._doubleValue = doubleValue;
|
|
4524
|
+
this._stringValue = stringValue;
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Returns the integer value of this symbol.
|
|
4528
|
+
*
|
|
4529
|
+
* @return {number|null} The integer value.
|
|
4530
|
+
*/
|
|
4531
|
+
get intValue() {
|
|
4532
|
+
return this._intValue;
|
|
4533
|
+
}
|
|
4534
|
+
/**
|
|
4535
|
+
* Returns the double value of this symbol.
|
|
4536
|
+
*
|
|
4537
|
+
* @return {number|null} The double value.
|
|
4538
|
+
*/
|
|
4539
|
+
get doubleValue() {
|
|
4540
|
+
return this._doubleValue;
|
|
4541
|
+
}
|
|
4542
|
+
/**
|
|
4543
|
+
* Returns the string value of this symbol.
|
|
4544
|
+
*
|
|
4545
|
+
* @return {string|null} The string value.
|
|
4546
|
+
*/
|
|
4547
|
+
get stringValue() {
|
|
4548
|
+
return this._stringValue;
|
|
4549
|
+
}
|
|
4550
|
+
/**
|
|
4551
|
+
* Retrieves the primary value of this symbol. The primary value is descriptive raw value.
|
|
4552
|
+
* It is either the string value, the integer value or the double value, prioritized in this order.
|
|
4553
|
+
*
|
|
4554
|
+
* @return {number|string|null} The primary value.
|
|
4555
|
+
*/
|
|
4556
|
+
toPrimaryValue() {
|
|
4557
|
+
if (null != this._stringValue) {
|
|
4558
|
+
return this._stringValue;
|
|
4559
|
+
} else if (null != this._intValue) {
|
|
4560
|
+
return this._intValue;
|
|
4561
|
+
} else if (null != this._doubleValue) {
|
|
4562
|
+
return this._doubleValue;
|
|
4563
|
+
} else {
|
|
4564
|
+
return null;
|
|
4565
|
+
}
|
|
4566
|
+
}
|
|
4567
|
+
/**
|
|
4568
|
+
* Converts the symbol to its byte representation.
|
|
4569
|
+
*
|
|
4570
|
+
* The kind is the one the symbol carries - an int, a double, a string, or a dual of an int or a
|
|
4571
|
+
* double with its text - so a symbol built with `fromDoubleValue(4)` stays a double. Each half is
|
|
4572
|
+
* checked before a byte is written, because `QvdSymbol`'s constructor checks nothing: an integer
|
|
4573
|
+
* outside int32 used to surface as a bare `RangeError` from `writeInt32LE`, a symbol holding an
|
|
4574
|
+
* integer and a double silently lost the double, a text with a NUL produced a symbol that ends
|
|
4575
|
+
* early, and a text with an unpaired surrogate was written with U+FFFD in its place.
|
|
4576
|
+
*
|
|
4577
|
+
* A half left `undefined` - `new QvdSymbol()`, or `new QvdSymbol(7)` - is absent, as it is to
|
|
4578
|
+
* `toPrimaryValue`. It used to be read as present: `new QvdSymbol(7)` threw a bare `TypeError` from
|
|
4579
|
+
* `Buffer.from`, and `new QvdSymbol(undefined, 4.5, '4.50')` was written as a dual of the integer 0.
|
|
4580
|
+
*
|
|
4581
|
+
* @return {Buffer} The byte representation of the symbol.
|
|
4582
|
+
* @throws {QvdValidationError} If the symbol holds both an integer and a double, holds nothing, or
|
|
4583
|
+
* holds a half no symbol can store. The message names the half.
|
|
4584
|
+
*/
|
|
4585
|
+
toByteRepresentation() {
|
|
4586
|
+
const intValue = this._intValue ?? null;
|
|
4587
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4588
|
+
const stringValue = this._stringValue ?? null;
|
|
4589
|
+
if (intValue !== null && doubleValue !== null) {
|
|
4590
|
+
throw new QvdValidationError("A symbol holds an integer or a double, not both", {
|
|
4591
|
+
intValue,
|
|
4592
|
+
doubleValue
|
|
4593
|
+
});
|
|
4594
|
+
}
|
|
4595
|
+
if (intValue === null && doubleValue === null && stringValue === null) {
|
|
4596
|
+
throw new QvdValidationError("The symbol does not contain any value.", {
|
|
4597
|
+
intValue,
|
|
4598
|
+
doubleValue,
|
|
4599
|
+
stringValue
|
|
4600
|
+
});
|
|
4601
|
+
}
|
|
4602
|
+
if (intValue !== null) {
|
|
4603
|
+
checkInteger(intValue, {});
|
|
4604
|
+
}
|
|
4605
|
+
if (doubleValue !== null) {
|
|
4606
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4607
|
+
}
|
|
4608
|
+
if (stringValue !== null) {
|
|
4609
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4610
|
+
}
|
|
4611
|
+
const number = intValue ?? doubleValue;
|
|
4612
|
+
const kind = number === null ? 4 : (intValue !== null ? 1 : 2) + (stringValue !== null ? 4 : 0);
|
|
4613
|
+
const buffer = Buffer.allocUnsafe(symbolByteLength(kind, number, stringValue));
|
|
4614
|
+
writeSymbol(buffer, 0, kind, number, stringValue);
|
|
4615
|
+
return buffer;
|
|
4616
|
+
}
|
|
4617
|
+
/**
|
|
4618
|
+
* Checks if this symbol is equal to another symbol.
|
|
4619
|
+
*
|
|
4620
|
+
* By shape rather than by class: another value is equal when its `intValue`, `doubleValue` and
|
|
4621
|
+
* `stringValue` are, compared with `===`, whichever copy of this library built it - `instanceof`
|
|
4622
|
+
* answers false for a symbol from the CommonJS build tested by the ESM one. A dual value, a `QvdDual`
|
|
4623
|
+
* or `{number, text}`, is equal to a dual symbol with the same number and text: a dual carries no
|
|
4624
|
+
* storage kind, so either kind matches.
|
|
4625
|
+
*
|
|
4626
|
+
* @param {*} value The object to compare with.
|
|
4627
|
+
* @return {boolean} True if the objects are equal, false otherwise.
|
|
4628
|
+
*/
|
|
4629
|
+
equals(value) {
|
|
4630
|
+
if (value === null || typeof value !== "object") {
|
|
4631
|
+
return false;
|
|
4632
|
+
}
|
|
4633
|
+
const intValue = this._intValue ?? null;
|
|
4634
|
+
const doubleValue = this._doubleValue ?? null;
|
|
4635
|
+
const stringValue = this._stringValue ?? null;
|
|
4636
|
+
const dual = asDual(value);
|
|
4637
|
+
if (dual !== null) {
|
|
4638
|
+
return stringValue !== null && stringValue === dual.text && intValue === null !== (doubleValue === null) && (intValue ?? doubleValue) === dual.number;
|
|
4639
|
+
}
|
|
4640
|
+
if (!("intValue" in value && "doubleValue" in value && "stringValue" in value)) {
|
|
4641
|
+
return false;
|
|
4642
|
+
}
|
|
4643
|
+
return intValue === (value.intValue ?? null) && doubleValue === (value.doubleValue ?? null) && stringValue === (value.stringValue ?? null);
|
|
4644
|
+
}
|
|
4645
|
+
/**
|
|
4646
|
+
* Constructs a pure integer value symbol.
|
|
4647
|
+
*
|
|
4648
|
+
* @param {number} intValue The integer value.
|
|
4649
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4650
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range.
|
|
4651
|
+
*/
|
|
4652
|
+
static fromIntValue(intValue) {
|
|
4653
|
+
checkInteger(intValue, {});
|
|
4654
|
+
return new _QvdSymbol(intValue, null, null);
|
|
4655
|
+
}
|
|
4656
|
+
/**
|
|
4657
|
+
* Constructs a pure double value symbol.
|
|
4658
|
+
*
|
|
4659
|
+
* @param {number} doubleValue The double value.
|
|
4660
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4661
|
+
* @throws {QvdValidationError} If the double is not a finite number.
|
|
4662
|
+
*/
|
|
4663
|
+
static fromDoubleValue(doubleValue) {
|
|
4664
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4665
|
+
return new _QvdSymbol(null, doubleValue, null);
|
|
4666
|
+
}
|
|
4667
|
+
/**
|
|
4668
|
+
* Constructs a pure string value symbol.
|
|
4669
|
+
*
|
|
4670
|
+
* @param {string} stringValue The string value.
|
|
4671
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4672
|
+
* @throws {QvdValidationError} If the string is not a string, or holds a NUL or an unpaired surrogate.
|
|
4673
|
+
*/
|
|
4674
|
+
static fromStringValue(stringValue) {
|
|
4675
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4676
|
+
return new _QvdSymbol(null, null, stringValue);
|
|
4677
|
+
}
|
|
4678
|
+
/**
|
|
4679
|
+
* Constructs a dual value symbol from an integer and a string value.
|
|
4680
|
+
*
|
|
4681
|
+
* @param {number} intValue The integer value.
|
|
4682
|
+
* @param {string} stringValue The string value.
|
|
4683
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4684
|
+
* @throws {QvdValidationError} If the integer is not an integer inside the int32 range, or the text
|
|
4685
|
+
* is not a string, or holds a NUL or an unpaired surrogate.
|
|
4686
|
+
*/
|
|
4687
|
+
static fromDualIntValue(intValue, stringValue) {
|
|
4688
|
+
checkInteger(intValue, {});
|
|
4689
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4690
|
+
return new _QvdSymbol(intValue, null, stringValue);
|
|
4691
|
+
}
|
|
4692
|
+
/**
|
|
4693
|
+
* Constructs a dual value symbol from a double and a string value.
|
|
4694
|
+
*
|
|
4695
|
+
* @param {number} doubleValue The double value.
|
|
4696
|
+
* @param {string} stringValue The string value.
|
|
4697
|
+
* @return {QvdSymbol} The constructed value symbol.
|
|
4698
|
+
* @throws {QvdValidationError} If the double is not a finite number, or the text is not a string,
|
|
4699
|
+
* or holds a NUL or an unpaired surrogate.
|
|
4700
|
+
*/
|
|
4701
|
+
static fromDualDoubleValue(doubleValue, stringValue) {
|
|
4702
|
+
checkNumber(doubleValue, "The double of a symbol", { half: "double" });
|
|
4703
|
+
checkText(stringValue, "The text of a symbol", { half: "text" });
|
|
4704
|
+
return new _QvdSymbol(null, doubleValue, stringValue);
|
|
4705
|
+
}
|
|
4706
|
+
};
|
|
4707
|
+
|
|
4708
|
+
// src/index.js
|
|
4709
|
+
init_QvdDual();
|
|
4710
|
+
|
|
4711
|
+
// src/util/qlikDate.js
|
|
4712
|
+
init_QvdErrors();
|
|
4713
|
+
init_cellRules();
|
|
4714
|
+
var QLIK_EPOCH_MS = Date.UTC(1899, 11, 30);
|
|
4715
|
+
var MS_PER_DAY = 864e5;
|
|
4716
|
+
var MAX_DATE_MS = 864e13;
|
|
4717
|
+
var MIN_SERIAL = (-MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4718
|
+
var MAX_SERIAL = (MAX_DATE_MS - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4719
|
+
function qlikSerialToDate(serial) {
|
|
4720
|
+
const dual = asDual(serial);
|
|
4721
|
+
const number = dual === null ? serial : dual.number;
|
|
4722
|
+
if (typeof number !== "number" || !Number.isFinite(number)) {
|
|
4723
|
+
throw new QvdValidationError("A Qlik date serial must be a finite number", {
|
|
4724
|
+
provided: describeType(number),
|
|
4725
|
+
type: typeof serial
|
|
4726
|
+
});
|
|
4727
|
+
}
|
|
4728
|
+
const ms = Math.round(QLIK_EPOCH_MS + number * MS_PER_DAY);
|
|
4729
|
+
if (!(Math.abs(ms) <= MAX_DATE_MS)) {
|
|
4730
|
+
throw new QvdValidationError(`A Qlik date serial of ${number} is outside the range a JavaScript Date can hold`, {
|
|
4731
|
+
serial: number,
|
|
4732
|
+
minSerial: MIN_SERIAL,
|
|
4733
|
+
maxSerial: MAX_SERIAL
|
|
4734
|
+
});
|
|
4735
|
+
}
|
|
4736
|
+
return new Date(ms);
|
|
4737
|
+
}
|
|
4738
|
+
__name(qlikSerialToDate, "qlikSerialToDate");
|
|
4739
|
+
function dateToQlikSerial(date) {
|
|
4740
|
+
const ms = types.isDate(date) ? Date.prototype.getTime.call(date) : Number.NaN;
|
|
4741
|
+
if (Number.isNaN(ms)) {
|
|
4742
|
+
throw new QvdValidationError("dateToQlikSerial needs a valid Date", {
|
|
4743
|
+
provided: describeType(date),
|
|
4744
|
+
type: typeof date
|
|
4745
|
+
});
|
|
4746
|
+
}
|
|
4747
|
+
return (ms - QLIK_EPOCH_MS) / MS_PER_DAY;
|
|
4748
|
+
}
|
|
4749
|
+
__name(dateToQlikSerial, "dateToQlikSerial");
|
|
4750
|
+
|
|
3357
4751
|
// src/index.js
|
|
3358
|
-
init_QvdSymbol();
|
|
3359
4752
|
init_QvdDataFrame();
|
|
3360
4753
|
init_QvdColumnTable();
|
|
3361
4754
|
init_QvdFileReader();
|
|
3362
4755
|
init_QvdFileWriter();
|
|
3363
4756
|
init_QvdErrors();
|
|
3364
4757
|
|
|
3365
|
-
export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError };
|
|
4758
|
+
export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdDual, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError, dateToQlikSerial, qlikSerialToDate };
|
|
3366
4759
|
//# sourceMappingURL=index.js.map
|
|
3367
4760
|
//# sourceMappingURL=index.js.map
|