qvdjs 1.0.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -1
- package/README.md +102 -32
- package/dist/index.cjs +745 -189
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +741 -185
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import
|
|
1
|
+
import fs2 from 'fs';
|
|
2
|
+
import path2 from 'path';
|
|
3
|
+
import assert3 from 'assert';
|
|
4
|
+
import crypto2 from 'crypto';
|
|
5
|
+
import { setTimeout } from 'timers/promises';
|
|
4
6
|
import xml2 from 'xml2js';
|
|
5
|
-
import assert2 from 'assert';
|
|
6
7
|
import os from 'os';
|
|
7
8
|
import v8 from 'v8';
|
|
8
9
|
import { types } from 'util';
|
|
@@ -33,9 +34,11 @@ var init_QvdErrors = __esm({
|
|
|
33
34
|
* @param {string} message The error message.
|
|
34
35
|
* @param {string} code The error code.
|
|
35
36
|
* @param {Object} [context={}] Additional context about the error.
|
|
37
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`, so a `cause` becomes `error.cause`: the
|
|
38
|
+
* error this one reports, such as the operating system's refusal behind a `QvdIOError`.
|
|
36
39
|
*/
|
|
37
|
-
constructor(message, code, context = {}) {
|
|
38
|
-
super(message);
|
|
40
|
+
constructor(message, code, context = {}, options = void 0) {
|
|
41
|
+
super(message, options);
|
|
39
42
|
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
40
43
|
this.code = code;
|
|
41
44
|
this.context = context;
|
|
@@ -51,9 +54,10 @@ var init_QvdErrors = __esm({
|
|
|
51
54
|
*
|
|
52
55
|
* @param {string} message The error message.
|
|
53
56
|
* @param {Object} [context={}] Additional context about the error.
|
|
57
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
54
58
|
*/
|
|
55
|
-
constructor(message, context = {}) {
|
|
56
|
-
super(message, "QVD_PARSE_ERROR", context);
|
|
59
|
+
constructor(message, context = {}, options = void 0) {
|
|
60
|
+
super(message, "QVD_PARSE_ERROR", context, options);
|
|
57
61
|
}
|
|
58
62
|
};
|
|
59
63
|
QvdValidationError = class extends QvdError {
|
|
@@ -65,9 +69,10 @@ var init_QvdErrors = __esm({
|
|
|
65
69
|
*
|
|
66
70
|
* @param {string} message The error message.
|
|
67
71
|
* @param {Object} [context={}] Additional context about the error.
|
|
72
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
68
73
|
*/
|
|
69
|
-
constructor(message, context = {}) {
|
|
70
|
-
super(message, "QVD_VALIDATION_ERROR", context);
|
|
74
|
+
constructor(message, context = {}, options = void 0) {
|
|
75
|
+
super(message, "QVD_VALIDATION_ERROR", context, options);
|
|
71
76
|
}
|
|
72
77
|
};
|
|
73
78
|
QvdIOError = class extends QvdError {
|
|
@@ -79,9 +84,10 @@ var init_QvdErrors = __esm({
|
|
|
79
84
|
*
|
|
80
85
|
* @param {string} message The error message.
|
|
81
86
|
* @param {Object} [context={}] Additional context about the error.
|
|
87
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
82
88
|
*/
|
|
83
|
-
constructor(message, context = {}) {
|
|
84
|
-
super(message, "QVD_IO_ERROR", context);
|
|
89
|
+
constructor(message, context = {}, options = void 0) {
|
|
90
|
+
super(message, "QVD_IO_ERROR", context, options);
|
|
85
91
|
}
|
|
86
92
|
};
|
|
87
93
|
QvdCorruptedError = class extends QvdError {
|
|
@@ -93,9 +99,10 @@ var init_QvdErrors = __esm({
|
|
|
93
99
|
*
|
|
94
100
|
* @param {string} message The error message.
|
|
95
101
|
* @param {Object} [context={}] Additional context about the error.
|
|
102
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
96
103
|
*/
|
|
97
|
-
constructor(message, context = {}) {
|
|
98
|
-
super(message, "QVD_CORRUPTED_ERROR", context);
|
|
104
|
+
constructor(message, context = {}, options = void 0) {
|
|
105
|
+
super(message, "QVD_CORRUPTED_ERROR", context, options);
|
|
99
106
|
}
|
|
100
107
|
};
|
|
101
108
|
QvdSecurityError = class extends QvdError {
|
|
@@ -107,9 +114,10 @@ var init_QvdErrors = __esm({
|
|
|
107
114
|
*
|
|
108
115
|
* @param {string} message The error message.
|
|
109
116
|
* @param {Object} [context={}] Additional context about the error.
|
|
117
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
110
118
|
*/
|
|
111
|
-
constructor(message, context = {}) {
|
|
112
|
-
super(message, "QVD_SECURITY_ERROR", context);
|
|
119
|
+
constructor(message, context = {}, options = void 0) {
|
|
120
|
+
super(message, "QVD_SECURITY_ERROR", context, options);
|
|
113
121
|
}
|
|
114
122
|
};
|
|
115
123
|
ERROR_NAMES.set(QvdError, "QvdError");
|
|
@@ -407,6 +415,28 @@ var init_QvdDual = __esm({
|
|
|
407
415
|
}
|
|
408
416
|
});
|
|
409
417
|
|
|
418
|
+
// src/util/optionTypes.js
|
|
419
|
+
function booleanOption(value, { option, file, whenUnset }) {
|
|
420
|
+
if (value === void 0 || value === null) {
|
|
421
|
+
return whenUnset;
|
|
422
|
+
}
|
|
423
|
+
if (typeof value !== "boolean") {
|
|
424
|
+
throw new QvdValidationError(`${option} must be true or false`, {
|
|
425
|
+
option,
|
|
426
|
+
provided: value,
|
|
427
|
+
type: typeof value,
|
|
428
|
+
file
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
return value;
|
|
432
|
+
}
|
|
433
|
+
var init_optionTypes = __esm({
|
|
434
|
+
"src/util/optionTypes.js"() {
|
|
435
|
+
init_QvdErrors();
|
|
436
|
+
__name(booleanOption, "booleanOption");
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
|
|
410
440
|
// src/util/readOptions.js
|
|
411
441
|
function requireRowCount(value, name, filePath) {
|
|
412
442
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
@@ -517,18 +547,7 @@ function normaliseDuals(value, filePath) {
|
|
|
517
547
|
return value;
|
|
518
548
|
}
|
|
519
549
|
function normaliseCoerceNumericStrings(value, filePath) {
|
|
520
|
-
|
|
521
|
-
return false;
|
|
522
|
-
}
|
|
523
|
-
if (typeof value !== "boolean") {
|
|
524
|
-
throw new QvdValidationError("coerceNumericStrings must be true or false", {
|
|
525
|
-
option: "coerceNumericStrings",
|
|
526
|
-
provided: value,
|
|
527
|
-
type: typeof value,
|
|
528
|
-
file: filePath
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
return value;
|
|
550
|
+
return booleanOption(value, { option: "coerceNumericStrings", file: filePath, whenUnset: false });
|
|
532
551
|
}
|
|
533
552
|
function readerOptionsFrom(options) {
|
|
534
553
|
return {
|
|
@@ -556,6 +575,7 @@ var DUAL_MODES;
|
|
|
556
575
|
var init_readOptions = __esm({
|
|
557
576
|
"src/util/readOptions.js"() {
|
|
558
577
|
init_QvdErrors();
|
|
578
|
+
init_optionTypes();
|
|
559
579
|
__name(requireRowCount, "requireRowCount");
|
|
560
580
|
__name(normaliseWindow, "normaliseWindow");
|
|
561
581
|
__name(resolveWindow, "resolveWindow");
|
|
@@ -699,24 +719,64 @@ var init_storedSymbols = __esm({
|
|
|
699
719
|
__name(sameValueZero, "sameValueZero");
|
|
700
720
|
}
|
|
701
721
|
});
|
|
722
|
+
function readDanglingLink(target) {
|
|
723
|
+
let link;
|
|
724
|
+
try {
|
|
725
|
+
link = fs2.lstatSync(target);
|
|
726
|
+
} catch {
|
|
727
|
+
return NOT_A_LINK;
|
|
728
|
+
}
|
|
729
|
+
if (!link.isSymbolicLink()) {
|
|
730
|
+
return NOT_A_LINK;
|
|
731
|
+
}
|
|
732
|
+
let destination;
|
|
733
|
+
try {
|
|
734
|
+
destination = fs2.readlinkSync(target);
|
|
735
|
+
} catch {
|
|
736
|
+
return UNREADABLE_LINK;
|
|
737
|
+
}
|
|
738
|
+
if (path2.isAbsolute(destination)) {
|
|
739
|
+
return destination;
|
|
740
|
+
}
|
|
741
|
+
try {
|
|
742
|
+
return path2.resolve(fs2.realpathSync(path2.dirname(target)), destination);
|
|
743
|
+
} catch {
|
|
744
|
+
return UNREADABLE_LINK;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
var MAX_LINK_HOPS, NOT_A_LINK, UNREADABLE_LINK;
|
|
748
|
+
var init_linkTarget = __esm({
|
|
749
|
+
"src/util/linkTarget.js"() {
|
|
750
|
+
MAX_LINK_HOPS = 40;
|
|
751
|
+
NOT_A_LINK = /* @__PURE__ */ Symbol("not a link");
|
|
752
|
+
UNREADABLE_LINK = /* @__PURE__ */ Symbol("unreadable link");
|
|
753
|
+
__name(readDanglingLink, "readDanglingLink");
|
|
754
|
+
}
|
|
755
|
+
});
|
|
702
756
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
703
757
|
const isCaseInsensitiveFS = process.platform === "win32";
|
|
704
758
|
const base = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
|
|
705
759
|
const target = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
|
|
706
|
-
const relative =
|
|
760
|
+
const relative = path2.relative(base, target);
|
|
707
761
|
if (relative === "") {
|
|
708
762
|
return true;
|
|
709
763
|
}
|
|
710
|
-
if (
|
|
764
|
+
if (path2.isAbsolute(relative)) {
|
|
711
765
|
return false;
|
|
712
766
|
}
|
|
713
|
-
return relative !== ".." && !relative.startsWith(`..${
|
|
767
|
+
return relative !== ".." && !relative.startsWith(`..${path2.sep}`);
|
|
714
768
|
}
|
|
715
769
|
function resolveDeepestExisting(target) {
|
|
716
770
|
let current = target;
|
|
771
|
+
let farEnd = target;
|
|
772
|
+
let walkedUp = false;
|
|
773
|
+
let followedWhileClimbing = false;
|
|
774
|
+
let hops = 0;
|
|
717
775
|
for (; ; ) {
|
|
718
776
|
try {
|
|
719
|
-
|
|
777
|
+
const deepest = fs2.realpathSync(current);
|
|
778
|
+
const canonicalFarEnd = walkedUp && !followedWhileClimbing ? path2.join(deepest, path2.relative(current, farEnd)) : farEnd;
|
|
779
|
+
return { deepest, exists: !walkedUp, farEnd: canonicalFarEnd };
|
|
720
780
|
} catch (error) {
|
|
721
781
|
const code = (
|
|
722
782
|
/** @type {{code?: string}} */
|
|
@@ -725,10 +785,27 @@ function resolveDeepestExisting(target) {
|
|
|
725
785
|
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
726
786
|
return null;
|
|
727
787
|
}
|
|
728
|
-
const
|
|
788
|
+
const followed = readDanglingLink(current);
|
|
789
|
+
if (followed === UNREADABLE_LINK) {
|
|
790
|
+
return REFUSED;
|
|
791
|
+
}
|
|
792
|
+
if (followed !== NOT_A_LINK) {
|
|
793
|
+
if (++hops > MAX_LINK_HOPS) {
|
|
794
|
+
return REFUSED;
|
|
795
|
+
}
|
|
796
|
+
current = followed;
|
|
797
|
+
if (walkedUp) {
|
|
798
|
+
followedWhileClimbing = true;
|
|
799
|
+
} else {
|
|
800
|
+
farEnd = followed;
|
|
801
|
+
}
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
const parent = path2.dirname(current);
|
|
729
805
|
if (parent === current) {
|
|
730
806
|
return null;
|
|
731
807
|
}
|
|
808
|
+
walkedUp = true;
|
|
732
809
|
current = parent;
|
|
733
810
|
}
|
|
734
811
|
}
|
|
@@ -736,32 +813,42 @@ function resolveDeepestExisting(target) {
|
|
|
736
813
|
function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
737
814
|
let baseStat;
|
|
738
815
|
try {
|
|
739
|
-
baseStat =
|
|
816
|
+
baseStat = fs2.statSync(fs2.realpathSync(resolvedBaseDir), { bigint: true });
|
|
740
817
|
} catch {
|
|
741
818
|
return null;
|
|
742
819
|
}
|
|
743
|
-
|
|
744
|
-
if (
|
|
820
|
+
const found = resolveDeepestExisting(resolvedPath);
|
|
821
|
+
if (found === REFUSED) {
|
|
822
|
+
return { contained: false, target: resolvedPath, stats: null };
|
|
823
|
+
}
|
|
824
|
+
if (found === null) {
|
|
745
825
|
return null;
|
|
746
826
|
}
|
|
747
|
-
|
|
827
|
+
const target = found.exists ? found.deepest : found.farEnd;
|
|
828
|
+
let stats = null;
|
|
829
|
+
let current = found.deepest;
|
|
830
|
+
for (let first = true; ; first = false) {
|
|
831
|
+
const ownIdentity = first && found.exists;
|
|
748
832
|
let stat;
|
|
749
833
|
try {
|
|
750
|
-
stat =
|
|
834
|
+
stat = ownIdentity ? fs2.lstatSync(current, { bigint: true }) : fs2.statSync(current, { bigint: true });
|
|
751
835
|
} catch {
|
|
752
836
|
return null;
|
|
753
837
|
}
|
|
838
|
+
if (ownIdentity) {
|
|
839
|
+
stats = stat;
|
|
840
|
+
}
|
|
754
841
|
if (stat.dev === baseStat.dev && stat.ino === baseStat.ino) {
|
|
755
|
-
return true;
|
|
842
|
+
return { contained: true, target, stats };
|
|
756
843
|
}
|
|
757
|
-
const parent =
|
|
844
|
+
const parent = path2.dirname(current);
|
|
758
845
|
if (parent === current) {
|
|
759
|
-
return false;
|
|
846
|
+
return { contained: false, target, stats };
|
|
760
847
|
}
|
|
761
848
|
current = parent;
|
|
762
849
|
}
|
|
763
850
|
}
|
|
764
|
-
function
|
|
851
|
+
function checkPath(filePath, allowedDir) {
|
|
765
852
|
if (typeof filePath !== "string" || filePath.length === 0) {
|
|
766
853
|
throw new QvdValidationError("filePath must be a non-empty string", {
|
|
767
854
|
provided: filePath,
|
|
@@ -780,11 +867,11 @@ function validatePath(filePath, allowedDir) {
|
|
|
780
867
|
reason: "null_byte"
|
|
781
868
|
});
|
|
782
869
|
}
|
|
783
|
-
const resolvedPath =
|
|
870
|
+
const resolvedPath = path2.resolve(filePath);
|
|
784
871
|
const baseDir = allowedDir || process.cwd();
|
|
785
|
-
const resolvedBaseDir =
|
|
872
|
+
const resolvedBaseDir = path2.resolve(baseDir);
|
|
786
873
|
const onDisk = isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath);
|
|
787
|
-
const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk;
|
|
874
|
+
const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk.contained;
|
|
788
875
|
if (!contained) {
|
|
789
876
|
throw new QvdSecurityError("Path traversal detected: Access denied", {
|
|
790
877
|
path: filePath,
|
|
@@ -796,15 +883,197 @@ function validatePath(filePath, allowedDir) {
|
|
|
796
883
|
check: onDisk === null ? "lexical" : "filesystem"
|
|
797
884
|
});
|
|
798
885
|
}
|
|
799
|
-
|
|
886
|
+
if (onDisk === null) {
|
|
887
|
+
return { path: resolvedPath, base: resolvedBaseDir, target: resolvedPath, stats: null, onDisk: false };
|
|
888
|
+
}
|
|
889
|
+
return { path: resolvedPath, base: resolvedBaseDir, target: onDisk.target, stats: onDisk.stats, onDisk: true };
|
|
800
890
|
}
|
|
891
|
+
var REFUSED;
|
|
801
892
|
var init_validatePath = __esm({
|
|
802
893
|
"src/util/validatePath.js"() {
|
|
803
894
|
init_QvdErrors();
|
|
895
|
+
init_linkTarget();
|
|
804
896
|
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
897
|
+
REFUSED = /* @__PURE__ */ Symbol("refused");
|
|
805
898
|
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
806
899
|
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
807
|
-
__name(
|
|
900
|
+
__name(checkPath, "checkPath");
|
|
901
|
+
}
|
|
902
|
+
});
|
|
903
|
+
|
|
904
|
+
// src/util/ioErrors.js
|
|
905
|
+
function isSystemError(error) {
|
|
906
|
+
const candidate = (
|
|
907
|
+
/** @type {{code?: unknown, syscall?: unknown}|null} */
|
|
908
|
+
error
|
|
909
|
+
);
|
|
910
|
+
return candidate !== null && typeof candidate === "object" && typeof candidate.syscall === "string" && typeof candidate.code === "string";
|
|
911
|
+
}
|
|
912
|
+
function asIoError(error, file, direction) {
|
|
913
|
+
if (!isSystemError(error)) {
|
|
914
|
+
return error;
|
|
915
|
+
}
|
|
916
|
+
const { message, syscall, code } = (
|
|
917
|
+
/** @type {{message: string, syscall: string, code: string}} */
|
|
918
|
+
error
|
|
919
|
+
);
|
|
920
|
+
return new QvdIOError(
|
|
921
|
+
`Could not ${direction} the QVD file: ${message}`,
|
|
922
|
+
{ file, operation: syscall, code },
|
|
923
|
+
{ cause: error }
|
|
924
|
+
);
|
|
925
|
+
}
|
|
926
|
+
function rethrowAsIoError(file, direction) {
|
|
927
|
+
return (error) => {
|
|
928
|
+
throw asIoError(error, file, direction);
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
async function closeAfter(handle, failed, use) {
|
|
932
|
+
let result;
|
|
933
|
+
try {
|
|
934
|
+
result = await use();
|
|
935
|
+
} catch (error) {
|
|
936
|
+
await handle.close().catch(() => {
|
|
937
|
+
});
|
|
938
|
+
throw error;
|
|
939
|
+
}
|
|
940
|
+
await handle.close().catch(failed);
|
|
941
|
+
return result;
|
|
942
|
+
}
|
|
943
|
+
var init_ioErrors = __esm({
|
|
944
|
+
"src/util/ioErrors.js"() {
|
|
945
|
+
init_QvdErrors();
|
|
946
|
+
__name(isSystemError, "isSystemError");
|
|
947
|
+
__name(asIoError, "asIoError");
|
|
948
|
+
__name(rethrowAsIoError, "rethrowAsIoError");
|
|
949
|
+
__name(closeAfter, "closeAfter");
|
|
950
|
+
}
|
|
951
|
+
});
|
|
952
|
+
function changedAfterCheck(checked, change) {
|
|
953
|
+
return new QvdSecurityError(`Path traversal detected: ${DESCRIPTIONS[change]}`, {
|
|
954
|
+
path: checked.path,
|
|
955
|
+
resolvedPath: checked.target,
|
|
956
|
+
allowedDir: checked.base,
|
|
957
|
+
reason: "changed_after_check",
|
|
958
|
+
change
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
async function openChecked(checked, purpose, failed, { nofollow = NOFOLLOW } = {}) {
|
|
962
|
+
assert3(purpose === "read" || checked.stats !== null, "A rewrite in place is of a file that exists.");
|
|
963
|
+
const noFollow = checked.onDisk ? nofollow : 0;
|
|
964
|
+
const flags = (purpose === "rewrite" ? O_WRONLY : O_RDONLY) | noFollow;
|
|
965
|
+
let handle;
|
|
966
|
+
try {
|
|
967
|
+
handle = await fs2.promises.open(checked.target, flags);
|
|
968
|
+
} catch (error) {
|
|
969
|
+
const { code } = (
|
|
970
|
+
/** @type {{code?: string}} */
|
|
971
|
+
error ?? {}
|
|
972
|
+
);
|
|
973
|
+
if (checked.onDisk && noFollow !== 0 && code !== void 0 && BECAME_A_LINK.has(code)) {
|
|
974
|
+
throw changedAfterCheck(checked, "became_a_symlink");
|
|
975
|
+
}
|
|
976
|
+
return failed(error);
|
|
977
|
+
}
|
|
978
|
+
const stats = checked.stats;
|
|
979
|
+
if (stats === null) {
|
|
980
|
+
if (checked.onDisk) {
|
|
981
|
+
await handle.close().catch(() => {
|
|
982
|
+
});
|
|
983
|
+
throw changedAfterCheck(checked, "appeared");
|
|
984
|
+
}
|
|
985
|
+
return handle;
|
|
986
|
+
}
|
|
987
|
+
try {
|
|
988
|
+
const opened = await handle.stat({ bigint: true }).catch(failed);
|
|
989
|
+
if (opened.dev !== stats.dev || opened.ino !== stats.ino) {
|
|
990
|
+
throw changedAfterCheck(checked, "replaced");
|
|
991
|
+
}
|
|
992
|
+
if (purpose === "rewrite" && stats.isFile()) {
|
|
993
|
+
await handle.truncate(0).catch(failed);
|
|
994
|
+
}
|
|
995
|
+
} catch (error) {
|
|
996
|
+
await handle.close().catch(() => {
|
|
997
|
+
});
|
|
998
|
+
throw error;
|
|
999
|
+
}
|
|
1000
|
+
return handle;
|
|
1001
|
+
}
|
|
1002
|
+
var NOFOLLOW, O_RDONLY, O_WRONLY, BECAME_A_LINK, DESCRIPTIONS;
|
|
1003
|
+
var init_openChecked = __esm({
|
|
1004
|
+
"src/util/openChecked.js"() {
|
|
1005
|
+
init_QvdErrors();
|
|
1006
|
+
NOFOLLOW = fs2.constants.O_NOFOLLOW ?? 0;
|
|
1007
|
+
({ O_RDONLY, O_WRONLY } = fs2.constants);
|
|
1008
|
+
BECAME_A_LINK = /* @__PURE__ */ new Set(["ELOOP", "EMLINK"]);
|
|
1009
|
+
DESCRIPTIONS = {
|
|
1010
|
+
became_a_symlink: "the file became a symbolic link after it was checked",
|
|
1011
|
+
replaced: "the file was replaced by a different one after it was checked",
|
|
1012
|
+
appeared: "a file appeared at the path after it was checked"
|
|
1013
|
+
};
|
|
1014
|
+
__name(changedAfterCheck, "changedAfterCheck");
|
|
1015
|
+
__name(openChecked, "openChecked");
|
|
1016
|
+
}
|
|
1017
|
+
});
|
|
1018
|
+
function firstBytesOf(name, budget) {
|
|
1019
|
+
if (Buffer.byteLength(name) <= budget) {
|
|
1020
|
+
return name;
|
|
1021
|
+
}
|
|
1022
|
+
let kept = "";
|
|
1023
|
+
let bytes = 0;
|
|
1024
|
+
for (const character of name) {
|
|
1025
|
+
const size = Buffer.byteLength(character);
|
|
1026
|
+
if (bytes + size > budget) {
|
|
1027
|
+
break;
|
|
1028
|
+
}
|
|
1029
|
+
kept += character;
|
|
1030
|
+
bytes += size;
|
|
1031
|
+
}
|
|
1032
|
+
return kept;
|
|
1033
|
+
}
|
|
1034
|
+
function temporaryPathFor(target) {
|
|
1035
|
+
const marks = `.qvdjs-${crypto2.randomBytes(8).toString("hex")}.tmp`;
|
|
1036
|
+
const name = `${firstBytesOf(path2.basename(target), MAX_NAME_BYTES - marks.length)}${marks}`;
|
|
1037
|
+
return path2.join(path2.dirname(target), name);
|
|
1038
|
+
}
|
|
1039
|
+
async function retrying(call, { platform = process.platform, delays = RETRY_DELAYS } = {}) {
|
|
1040
|
+
for (let attempt = 0; ; attempt++) {
|
|
1041
|
+
try {
|
|
1042
|
+
return await call();
|
|
1043
|
+
} catch (error) {
|
|
1044
|
+
const { code } = (
|
|
1045
|
+
/** @type {{code?: string}} */
|
|
1046
|
+
error ?? {}
|
|
1047
|
+
);
|
|
1048
|
+
if (platform !== "win32" || attempt >= delays.length || code === void 0 || !IN_USE.has(code)) {
|
|
1049
|
+
throw error;
|
|
1050
|
+
}
|
|
1051
|
+
await setTimeout(delays[attempt]);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
function renameOver(from, to, options) {
|
|
1056
|
+
return retrying(() => fs2.promises.rename(from, to), options);
|
|
1057
|
+
}
|
|
1058
|
+
async function removeTemporary(file, options) {
|
|
1059
|
+
try {
|
|
1060
|
+
await retrying(() => fs2.promises.rm(file, { force: true }), options);
|
|
1061
|
+
return true;
|
|
1062
|
+
} catch {
|
|
1063
|
+
return false;
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
var MAX_NAME_BYTES, IN_USE, RETRY_DELAYS;
|
|
1067
|
+
var init_replaceFile = __esm({
|
|
1068
|
+
"src/util/replaceFile.js"() {
|
|
1069
|
+
MAX_NAME_BYTES = 255;
|
|
1070
|
+
__name(firstBytesOf, "firstBytesOf");
|
|
1071
|
+
__name(temporaryPathFor, "temporaryPathFor");
|
|
1072
|
+
IN_USE = /* @__PURE__ */ new Set(["EPERM", "EACCES", "EBUSY"]);
|
|
1073
|
+
RETRY_DELAYS = [10, 20, 40, 80, 160, 320];
|
|
1074
|
+
__name(retrying, "retrying");
|
|
1075
|
+
__name(renameOver, "renameOver");
|
|
1076
|
+
__name(removeTemporary, "removeTemporary");
|
|
808
1077
|
}
|
|
809
1078
|
});
|
|
810
1079
|
|
|
@@ -817,8 +1086,12 @@ function fieldGeometry(bitOffset, bitWidth) {
|
|
|
817
1086
|
byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
|
|
818
1087
|
};
|
|
819
1088
|
}
|
|
820
|
-
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
|
|
1089
|
+
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
|
|
1090
|
+
const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
|
|
821
1091
|
if (bitWidth === 0) {
|
|
1092
|
+
if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
|
|
1093
|
+
refuse2(bounds, 0, bias, bias);
|
|
1094
|
+
}
|
|
822
1095
|
out.fill(bias, 0, rowCount);
|
|
823
1096
|
return out;
|
|
824
1097
|
}
|
|
@@ -832,10 +1105,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
|
|
|
832
1105
|
if (byteCount > 2) acc += buffer[base + 2] * 65536;
|
|
833
1106
|
if (byteCount > 3) acc += buffer[base + 3] * 16777216;
|
|
834
1107
|
if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
|
|
835
|
-
|
|
1108
|
+
const index = Math.floor(acc / divisor) % modulus + bias;
|
|
1109
|
+
if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
|
|
1110
|
+
refuse2(bounds, row, index, bias);
|
|
1111
|
+
}
|
|
1112
|
+
out[row] = index;
|
|
836
1113
|
}
|
|
837
1114
|
return out;
|
|
838
1115
|
}
|
|
1116
|
+
function refuse2(bounds, row, index, bias) {
|
|
1117
|
+
throw new QvdCorruptedError("Symbol index out of range", {
|
|
1118
|
+
field: bounds.field,
|
|
1119
|
+
row: bounds.firstRow + row,
|
|
1120
|
+
symbolIndex: index,
|
|
1121
|
+
symbolCount: bounds.symbolCount,
|
|
1122
|
+
bias,
|
|
1123
|
+
file: bounds.file,
|
|
1124
|
+
stage: "parseIndexTable"
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
839
1127
|
function writeBitField(buffer, recordBase, geometry, value) {
|
|
840
1128
|
const { byteStart, shift, byteCount } = geometry;
|
|
841
1129
|
if (byteCount === 0) {
|
|
@@ -852,10 +1140,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
|
|
|
852
1140
|
var MAX_BIT_WIDTH, POW2;
|
|
853
1141
|
var init_bitUtils = __esm({
|
|
854
1142
|
"src/util/bitUtils.js"() {
|
|
1143
|
+
init_QvdErrors();
|
|
855
1144
|
MAX_BIT_WIDTH = 31;
|
|
856
1145
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
857
1146
|
__name(fieldGeometry, "fieldGeometry");
|
|
858
1147
|
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
1148
|
+
__name(refuse2, "refuse");
|
|
859
1149
|
__name(writeBitField, "writeBitField");
|
|
860
1150
|
}
|
|
861
1151
|
});
|
|
@@ -1134,11 +1424,15 @@ function resetContradictedNumberFormat(numberFormat, facts) {
|
|
|
1134
1424
|
}
|
|
1135
1425
|
return numberFormat;
|
|
1136
1426
|
}
|
|
1137
|
-
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, QvdFileWriter;
|
|
1427
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, WRITE_CHUNK_SIZE, QvdFileWriter;
|
|
1138
1428
|
var init_QvdFileWriter = __esm({
|
|
1139
1429
|
"src/QvdFileWriter.js"() {
|
|
1140
1430
|
init_QvdErrors();
|
|
1141
1431
|
init_validatePath();
|
|
1432
|
+
init_ioErrors();
|
|
1433
|
+
init_openChecked();
|
|
1434
|
+
init_replaceFile();
|
|
1435
|
+
init_optionTypes();
|
|
1142
1436
|
init_bitUtils();
|
|
1143
1437
|
init_cellRules();
|
|
1144
1438
|
init_symbolBytes();
|
|
@@ -1165,6 +1459,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1165
1459
|
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1166
1460
|
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1167
1461
|
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1462
|
+
WRITE_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
1168
1463
|
QvdFileWriter = class {
|
|
1169
1464
|
static {
|
|
1170
1465
|
__name(this, "QvdFileWriter");
|
|
@@ -1181,10 +1476,30 @@ var init_QvdFileWriter = __esm({
|
|
|
1181
1476
|
* an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
|
|
1182
1477
|
* empty value falls back to the working directory rather than removing the restriction.
|
|
1183
1478
|
* @param {Function} [options.onProgress] Optional progress callback function.
|
|
1479
|
+
* @param {boolean} [options.atomic=true] Whether to replace the destination rather than rewrite it:
|
|
1480
|
+
* the file is built beside it under a temporary name and renamed over it, so a failure leaves the
|
|
1481
|
+
* previous file exactly as it was and nothing ever reads a part-written one. False rewrites the
|
|
1482
|
+
* destination in place, which is what every version before 2.0.0 did: that keeps the file's
|
|
1483
|
+
* identity - hard links, its owner, permissions set on the file itself - needs no room for two
|
|
1484
|
+
* copies at once and no permission to create files in the directory, and destroys the previous
|
|
1485
|
+
* file the moment the write begins. A file that does not exist yet is renamed into place either
|
|
1486
|
+
* way, since there is nothing to rewrite - see `_destination`.
|
|
1487
|
+
* @param {boolean} [options.fsync=true] Whether to wait for the file's contents to reach the disk
|
|
1488
|
+
* before the write is finished. It is what carries an atomic write's promise through a power
|
|
1489
|
+
* loss - the contents are on the disk before anything is renamed, so a crash leaves the previous
|
|
1490
|
+
* file or the new one and never a damaged one - and it is the only thing that reports a failing
|
|
1491
|
+
* disk's deferred error rather than losing it. The rename itself is not flushed, so a crash just
|
|
1492
|
+
* after the call can still lose the replacement, or a file that did not exist before. False
|
|
1493
|
+
* resolves as soon as the operating system has accepted the bytes, which is faster and is what
|
|
1494
|
+
* every version before 2.0.0 did.
|
|
1184
1495
|
*/
|
|
1185
1496
|
constructor(filePath, df, options = {}) {
|
|
1186
|
-
const { allowedDir, onProgress } = options;
|
|
1187
|
-
|
|
1497
|
+
const { allowedDir, onProgress, atomic, fsync } = options;
|
|
1498
|
+
const checked = checkPath(filePath, allowedDir);
|
|
1499
|
+
this._path = checked.path;
|
|
1500
|
+
this._allowedDir = checked.base;
|
|
1501
|
+
this._atomic = booleanOption(atomic, { option: "atomic", file: this._path, whenUnset: true });
|
|
1502
|
+
this._fsync = booleanOption(fsync, { option: "fsync", file: this._path, whenUnset: true });
|
|
1188
1503
|
this._df = df;
|
|
1189
1504
|
this._onProgress = onProgress;
|
|
1190
1505
|
this._header = null;
|
|
@@ -1220,22 +1535,168 @@ var init_QvdFileWriter = __esm({
|
|
|
1220
1535
|
* Writes the data to the QVD file.
|
|
1221
1536
|
*/
|
|
1222
1537
|
async _writeData() {
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1538
|
+
assert3(this._header, "The QVD file header has not been parsed.");
|
|
1539
|
+
assert3(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
|
|
1540
|
+
assert3(this._indexBuffer, "The QVD file index table has not been parsed.");
|
|
1226
1541
|
this._emitProgress("write", 0, 1);
|
|
1227
1542
|
const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
|
|
1228
|
-
|
|
1543
|
+
const failed = rethrowAsIoError(this._path, "write");
|
|
1544
|
+
const destination = this._destination();
|
|
1545
|
+
if (destination.replace) {
|
|
1546
|
+
await this._replaceFile(destination, headerBuffer, failed);
|
|
1547
|
+
} else {
|
|
1548
|
+
await this._writeInPlace(destination, headerBuffer, failed);
|
|
1549
|
+
}
|
|
1550
|
+
this._emitProgress("write", 1, 1);
|
|
1551
|
+
}
|
|
1552
|
+
/**
|
|
1553
|
+
* What is at the destination, and therefore how it is to be written.
|
|
1554
|
+
*
|
|
1555
|
+
* Decided by one containment check, made now - immediately before anything is opened - and by
|
|
1556
|
+
* nothing else: the file that check approved is the file written, in either mode, and what the
|
|
1557
|
+
* check found there is what decides how.
|
|
1558
|
+
*
|
|
1559
|
+
* It used to stat and resolve the path again for itself, after the check. That second resolution
|
|
1560
|
+
* was #247 in the atomic write: a destination swapped for a symlink after the check was followed by
|
|
1561
|
+
* it, so the temporary file was built beside a file outside allowedDir and renamed over it. A check
|
|
1562
|
+
* reports where it went, so nothing here has to go there again.
|
|
1563
|
+
*
|
|
1564
|
+
* @return {{checked: import('./util/validatePath.js').CheckedPath, path: string,
|
|
1565
|
+
* existing: import('fs').BigIntStats|null, replace: boolean, sync: boolean}} The check, the file it
|
|
1566
|
+
* approved, what was there, whether to replace it rather than rewrite it, and whether to flush it.
|
|
1567
|
+
* @private
|
|
1568
|
+
*/
|
|
1569
|
+
_destination() {
|
|
1570
|
+
const checked = checkPath(this._path, this._allowedDir);
|
|
1571
|
+
const existing = checked.stats;
|
|
1572
|
+
const special = existing !== null && !existing.isFile();
|
|
1573
|
+
const replace = (this._atomic || existing === null) && !special;
|
|
1574
|
+
return {
|
|
1575
|
+
checked,
|
|
1576
|
+
path: checked.target,
|
|
1577
|
+
existing,
|
|
1578
|
+
replace,
|
|
1579
|
+
sync: this._fsync && !special
|
|
1580
|
+
};
|
|
1581
|
+
}
|
|
1582
|
+
/**
|
|
1583
|
+
* Rewrites the destination where it stands - the way every version before 2.0.0 wrote.
|
|
1584
|
+
*
|
|
1585
|
+
* The file is emptied as the write begins, so from there until the last byte is written there is no
|
|
1586
|
+
* previous version left: a failure part-way through leaves a stub that no read of its rows survives,
|
|
1587
|
+
* and anything reading the path meanwhile sees however much of the new file has arrived.
|
|
1588
|
+
*
|
|
1589
|
+
* It is emptied by `openChecked` rather than by opening with `'w'`, and later than `'w'` would: once
|
|
1590
|
+
* the descriptor is known to be the file the check approved. `'w'` empties whatever the open reaches,
|
|
1591
|
+
* which is what let #247 truncate a file outside allowedDir through a symlink swapped in after the
|
|
1592
|
+
* check. The file that results is the same.
|
|
1593
|
+
*
|
|
1594
|
+
* @param {{checked: import('./util/validatePath.js').CheckedPath, sync: boolean}} destination Where
|
|
1595
|
+
* to write, from `_destination`.
|
|
1596
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1597
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1598
|
+
* @private
|
|
1599
|
+
*/
|
|
1600
|
+
async _writeInPlace(destination, headerBuffer, failed) {
|
|
1601
|
+
const fd = await openChecked(destination.checked, "rewrite", failed);
|
|
1602
|
+
await closeAfter(fd, failed, async () => {
|
|
1603
|
+
await this._writeParts(fd, headerBuffer, failed);
|
|
1604
|
+
if (destination.sync) {
|
|
1605
|
+
await fd.sync().catch(failed);
|
|
1606
|
+
}
|
|
1607
|
+
});
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* Builds the file beside the destination and renames it over it.
|
|
1611
|
+
*
|
|
1612
|
+
* Nothing touches the destination until the rename, which either replaces it or leaves it as it was,
|
|
1613
|
+
* so a write that fails at any step - a full disk, a process killed, an error from the disk itself -
|
|
1614
|
+
* costs the temporary file and nothing else. A reader of the path gets the previous file or the new
|
|
1615
|
+
* one, never a part of either, which is the other half of what the old behaviour could not promise.
|
|
1616
|
+
*
|
|
1617
|
+
* @param {{path: string, existing: import('fs').BigIntStats|null, sync: boolean}} destination Where to
|
|
1618
|
+
* write, from `_destination`.
|
|
1619
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1620
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1621
|
+
* @private
|
|
1622
|
+
*/
|
|
1623
|
+
async _replaceFile(destination, headerBuffer, failed) {
|
|
1624
|
+
const temporary = temporaryPathFor(destination.path);
|
|
1625
|
+
const mode = destination.existing === null ? void 0 : 384;
|
|
1626
|
+
const fd = await fs2.promises.open(temporary, "wx", mode).catch(failed);
|
|
1229
1627
|
try {
|
|
1230
|
-
fd
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1628
|
+
await closeAfter(fd, failed, async () => {
|
|
1629
|
+
await this._writeParts(fd, headerBuffer, failed);
|
|
1630
|
+
if (destination.existing !== null && process.platform !== "win32") {
|
|
1631
|
+
await fd.chmod(Number(destination.existing.mode) & 511).catch(failed);
|
|
1632
|
+
}
|
|
1633
|
+
if (destination.sync) {
|
|
1634
|
+
await fd.sync().catch(failed);
|
|
1635
|
+
}
|
|
1636
|
+
});
|
|
1637
|
+
await renameOver(temporary, destination.path).catch(failed);
|
|
1638
|
+
} catch (error) {
|
|
1639
|
+
const removed = await removeTemporary(temporary);
|
|
1640
|
+
const context = (
|
|
1641
|
+
/** @type {{context?: Record<string, unknown>}} */
|
|
1642
|
+
error?.context
|
|
1643
|
+
);
|
|
1644
|
+
if (!removed && context !== null && typeof context === "object" && Object.isExtensible(context)) {
|
|
1645
|
+
context.temporaryFile = temporary;
|
|
1238
1646
|
}
|
|
1647
|
+
throw error;
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
/**
|
|
1651
|
+
* Writes the three parts of a QVD, in their order, into an open file.
|
|
1652
|
+
*
|
|
1653
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
1654
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1655
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1656
|
+
* @private
|
|
1657
|
+
*/
|
|
1658
|
+
async _writeParts(fd, headerBuffer, failed) {
|
|
1659
|
+
await this._writeRange(fd, headerBuffer, 0, failed);
|
|
1660
|
+
await this._writeRange(fd, this._symbolBuffer, headerBuffer.length, failed);
|
|
1661
|
+
await this._writeRange(fd, this._indexBuffer, headerBuffer.length + this._symbolBuffer.length, failed);
|
|
1662
|
+
}
|
|
1663
|
+
/**
|
|
1664
|
+
* Writes the whole of one buffer into the file, starting at `filePosition`.
|
|
1665
|
+
*
|
|
1666
|
+
* A write can resolve having written less than it was given, and that is all a disk that fills
|
|
1667
|
+
* part-way through one says. libuv retries a short write(2) itself, and when the retry fails it
|
|
1668
|
+
* returns the bytes that did land instead of the error - `uv__fs_write_all` in its `src/unix/fs.c`,
|
|
1669
|
+
* and `fs__write` on Windows does the same - so Node resolves with a short `bytesWritten` and never
|
|
1670
|
+
* rejects. Taking that for the whole write is how `toQvd()` resolved on a full disk and left a
|
|
1671
|
+
* truncated file: the index table, the last of the three writes, stopped part-way, and no later call
|
|
1672
|
+
* asked the disk again.
|
|
1673
|
+
*
|
|
1674
|
+
* So the rest is written until there is none, and it is the next call that reports the failure: the
|
|
1675
|
+
* disk refuses it outright, and that rejection is a QvdIOError with the system's code, ENOSPC, like
|
|
1676
|
+
* any other refused write. A write that stores nothing and reports nothing would repeat forever, so
|
|
1677
|
+
* it is a failure too, the one QvdIOError with no system code to carry.
|
|
1678
|
+
*
|
|
1679
|
+
* In bounded chunks as well, because Node refuses a single write of 2 GiB or more.
|
|
1680
|
+
*
|
|
1681
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
1682
|
+
* @param {Buffer} buffer What to write.
|
|
1683
|
+
* @param {number} filePosition Where in the file it starts.
|
|
1684
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler, which every call
|
|
1685
|
+
* goes through.
|
|
1686
|
+
* @private
|
|
1687
|
+
*/
|
|
1688
|
+
async _writeRange(fd, buffer, filePosition, failed) {
|
|
1689
|
+
let done = 0;
|
|
1690
|
+
while (done < buffer.length) {
|
|
1691
|
+
const length = Math.min(WRITE_CHUNK_SIZE, buffer.length - done);
|
|
1692
|
+
const { bytesWritten } = await fd.write(buffer, done, length, filePosition + done).catch(failed);
|
|
1693
|
+
if (!(bytesWritten > 0)) {
|
|
1694
|
+
throw new QvdIOError(
|
|
1695
|
+
`Could not write the QVD file: nothing was written at byte ${filePosition + done}, and the operating system reported no error.`,
|
|
1696
|
+
{ file: this._path, operation: "write", filePosition: filePosition + done }
|
|
1697
|
+
);
|
|
1698
|
+
}
|
|
1699
|
+
done += bytesWritten;
|
|
1239
1700
|
}
|
|
1240
1701
|
}
|
|
1241
1702
|
/**
|
|
@@ -1247,13 +1708,13 @@ var init_QvdFileWriter = __esm({
|
|
|
1247
1708
|
const existingMetadata = this._df.metadata;
|
|
1248
1709
|
const baseMetadata = existingMetadata ? {
|
|
1249
1710
|
QvBuildNo: existingMetadata.QvBuildNo || 50667,
|
|
1250
|
-
CreatorDoc: existingMetadata.CreatorDoc ||
|
|
1711
|
+
CreatorDoc: existingMetadata.CreatorDoc || crypto2.randomUUID(),
|
|
1251
1712
|
CreateUtcTime: existingMetadata.CreateUtcTime || creationDate,
|
|
1252
1713
|
SourceCreateUtcTime: existingMetadata.SourceCreateUtcTime || "",
|
|
1253
1714
|
SourceFileUtcTime: existingMetadata.SourceFileUtcTime || "",
|
|
1254
1715
|
SourceFileSize: existingMetadata.SourceFileSize || -1,
|
|
1255
1716
|
StaleUtcTime: existingMetadata.StaleUtcTime || "",
|
|
1256
|
-
TableName: existingMetadata.TableName ||
|
|
1717
|
+
TableName: existingMetadata.TableName || path2.basename(this._path, path2.extname(this._path)),
|
|
1257
1718
|
Compression: existingMetadata.Compression || "",
|
|
1258
1719
|
Comment: existingMetadata.Comment || "",
|
|
1259
1720
|
EncryptionInfo: existingMetadata.EncryptionInfo || "",
|
|
@@ -1267,13 +1728,13 @@ var init_QvdFileWriter = __esm({
|
|
|
1267
1728
|
}
|
|
1268
1729
|
} : {
|
|
1269
1730
|
QvBuildNo: 50667,
|
|
1270
|
-
CreatorDoc:
|
|
1731
|
+
CreatorDoc: crypto2.randomUUID(),
|
|
1271
1732
|
CreateUtcTime: creationDate,
|
|
1272
1733
|
SourceCreateUtcTime: "",
|
|
1273
1734
|
SourceFileUtcTime: "",
|
|
1274
1735
|
SourceFileSize: -1,
|
|
1275
1736
|
StaleUtcTime: "",
|
|
1276
|
-
TableName:
|
|
1737
|
+
TableName: path2.basename(this._path, path2.extname(this._path)),
|
|
1277
1738
|
Compression: "",
|
|
1278
1739
|
Comment: "",
|
|
1279
1740
|
EncryptionInfo: "",
|
|
@@ -1524,7 +1985,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1524
1985
|
const key = keys[slot];
|
|
1525
1986
|
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1526
1987
|
}
|
|
1527
|
-
|
|
1988
|
+
assert3(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
1528
1989
|
columnBuffers.push(columnBuffer);
|
|
1529
1990
|
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1530
1991
|
this._symbolCounts?.push(keys.length);
|
|
@@ -1563,9 +2024,9 @@ var init_QvdFileWriter = __esm({
|
|
|
1563
2024
|
* @private
|
|
1564
2025
|
*/
|
|
1565
2026
|
_buildIndexTable() {
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
2027
|
+
assert3(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
2028
|
+
assert3(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
2029
|
+
assert3(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
1569
2030
|
this._indexTableMetadata = [];
|
|
1570
2031
|
const columns = this._df.columns;
|
|
1571
2032
|
const data = this._df.data;
|
|
@@ -2118,6 +2579,15 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2118
2579
|
stage: "parseIndexTable"
|
|
2119
2580
|
});
|
|
2120
2581
|
}
|
|
2582
|
+
if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
|
|
2583
|
+
throw new QvdCorruptedError("Bias out of range", {
|
|
2584
|
+
field: field["FieldName"],
|
|
2585
|
+
bias,
|
|
2586
|
+
bitWidth,
|
|
2587
|
+
file: filePath,
|
|
2588
|
+
stage: "parseIndexTable"
|
|
2589
|
+
});
|
|
2590
|
+
}
|
|
2121
2591
|
const recordSizeInBits = recordSize * 8;
|
|
2122
2592
|
if (bitOffset + bitWidth > recordSizeInBits) {
|
|
2123
2593
|
throw new QvdCorruptedError("Bit field extends beyond record size", {
|
|
@@ -2245,7 +2715,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2245
2715
|
}
|
|
2246
2716
|
return { numbers, texts };
|
|
2247
2717
|
}
|
|
2248
|
-
|
|
2718
|
+
function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
|
|
2719
|
+
return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
|
|
2720
|
+
}
|
|
2721
|
+
var MAX_TEXT_BYTES, DECODE_NOTHING;
|
|
2249
2722
|
var init_symbolParser = __esm({
|
|
2250
2723
|
"src/util/symbolParser.js"() {
|
|
2251
2724
|
init_QvdErrors();
|
|
@@ -2253,6 +2726,8 @@ var init_symbolParser = __esm({
|
|
|
2253
2726
|
__name(textEnd, "textEnd");
|
|
2254
2727
|
__name(overflow, "overflow");
|
|
2255
2728
|
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2729
|
+
DECODE_NOTHING = /* @__PURE__ */ new Set();
|
|
2730
|
+
__name(countFieldSymbols, "countFieldSymbols");
|
|
2256
2731
|
}
|
|
2257
2732
|
});
|
|
2258
2733
|
|
|
@@ -2684,9 +3159,9 @@ var init_QvdColumnTable = __esm({
|
|
|
2684
3159
|
* read switches to two-pass filtering.
|
|
2685
3160
|
* @return {Promise<QvdColumnTable>} The file, as columns.
|
|
2686
3161
|
*/
|
|
2687
|
-
static async fromQvd(
|
|
3162
|
+
static async fromQvd(path5, options = {}) {
|
|
2688
3163
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2689
|
-
const reader = new QvdFileReader2(
|
|
3164
|
+
const reader = new QvdFileReader2(path5, {
|
|
2690
3165
|
...readerOptionsFrom(options),
|
|
2691
3166
|
// This read builds no rows, so the memory guard must not charge it for them. A columnar
|
|
2692
3167
|
// read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
|
|
@@ -2754,22 +3229,26 @@ var QvdFileReader_exports = {};
|
|
|
2754
3229
|
__export(QvdFileReader_exports, {
|
|
2755
3230
|
QvdFileReader: () => QvdFileReader
|
|
2756
3231
|
});
|
|
2757
|
-
function
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
3232
|
+
async function* chunksFrom(handle, chunkSize, failed) {
|
|
3233
|
+
let position = 0;
|
|
3234
|
+
for (; ; ) {
|
|
3235
|
+
const buffer = Buffer.alloc(chunkSize);
|
|
3236
|
+
const { bytesRead } = await handle.read(buffer, 0, chunkSize, position).catch(failed);
|
|
3237
|
+
if (bytesRead === 0) {
|
|
3238
|
+
return;
|
|
3239
|
+
}
|
|
3240
|
+
yield bytesRead === chunkSize ? buffer : Buffer.from(buffer.subarray(0, bytesRead));
|
|
3241
|
+
position += bytesRead;
|
|
3242
|
+
}
|
|
2766
3243
|
}
|
|
2767
|
-
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, QvdFileReader;
|
|
3244
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST, QvdFileReader;
|
|
2768
3245
|
var init_QvdFileReader = __esm({
|
|
2769
3246
|
"src/QvdFileReader.js"() {
|
|
2770
3247
|
init_QvdDataFrame();
|
|
2771
3248
|
init_QvdErrors();
|
|
2772
3249
|
init_validatePath();
|
|
3250
|
+
init_openChecked();
|
|
3251
|
+
init_ioErrors();
|
|
2773
3252
|
init_bitUtils();
|
|
2774
3253
|
init_memoryUtils();
|
|
2775
3254
|
init_validationUtils();
|
|
@@ -2780,7 +3259,8 @@ var init_QvdFileReader = __esm({
|
|
|
2780
3259
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
2781
3260
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
2782
3261
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
2783
|
-
|
|
3262
|
+
COUNT_SYMBOLS_PAST = 65536;
|
|
3263
|
+
__name(chunksFrom, "chunksFrom");
|
|
2784
3264
|
QvdFileReader = class {
|
|
2785
3265
|
static {
|
|
2786
3266
|
__name(this, "QvdFileReader");
|
|
@@ -2841,7 +3321,9 @@ var init_QvdFileReader = __esm({
|
|
|
2841
3321
|
signal
|
|
2842
3322
|
} = options;
|
|
2843
3323
|
this._materialisesRows = materialisesRows;
|
|
2844
|
-
|
|
3324
|
+
const checked = checkPath(filePath, allowedDir);
|
|
3325
|
+
this._path = checked.path;
|
|
3326
|
+
this._allowedDir = checked.base;
|
|
2845
3327
|
this._duals = normaliseDuals(duals, this._path);
|
|
2846
3328
|
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
2847
3329
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
@@ -2926,18 +3408,21 @@ var init_QvdFileReader = __esm({
|
|
|
2926
3408
|
* two-pass path exist for.
|
|
2927
3409
|
*
|
|
2928
3410
|
* Algorithm for a windowed read:
|
|
2929
|
-
* 1.
|
|
3411
|
+
* 1. Read the file a chunk at a time until the XML header delimiter is found
|
|
2930
3412
|
* 2. Parse header to determine symbol table and index table locations
|
|
2931
3413
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
2932
|
-
* 4. Read only those calculated bytes
|
|
3414
|
+
* 4. Read only those calculated bytes, by position
|
|
2933
3415
|
* 5. Rest of parsing proceeds normally with limited data
|
|
2934
3416
|
*
|
|
2935
3417
|
* WHY THIS APPROACH:
|
|
2936
3418
|
* - Symbol table must be fully loaded (contains all unique values)
|
|
2937
3419
|
* - Index table can be partially loaded (only rows we need)
|
|
2938
|
-
* -
|
|
3420
|
+
* - Reading chunks to find the header is efficient for unknown header sizes
|
|
2939
3421
|
* - Direct byte-range reading for remaining data is fastest
|
|
2940
3422
|
*
|
|
3423
|
+
* All of it goes through one handle, opened once, on the file the containment check approved. See
|
|
3424
|
+
* `chunksFrom` and `openChecked` for why a read no longer opens the path more than once.
|
|
3425
|
+
*
|
|
2941
3426
|
* A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
|
|
2942
3427
|
* table from the front of the file, and the window's records from wherever they sit. The bytes
|
|
2943
3428
|
* between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
|
|
@@ -2955,49 +3440,49 @@ var init_QvdFileReader = __esm({
|
|
|
2955
3440
|
async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
|
|
2956
3441
|
this._throwIfAborted();
|
|
2957
3442
|
this._emitProgress("read", 0, 1);
|
|
3443
|
+
const failed = rethrowAsIoError(this._path, "read");
|
|
3444
|
+
const handle = await openChecked(checkPath(this._path, this._allowedDir), "read", failed);
|
|
3445
|
+
await closeAfter(handle, failed, () => this._readFrom(handle, window, headerOnly, liveRows, failed));
|
|
3446
|
+
}
|
|
3447
|
+
/**
|
|
3448
|
+
* Reads what `_readData` was asked for, through the handle it opened.
|
|
3449
|
+
*
|
|
3450
|
+
* @param {import('fs/promises').FileHandle} handle The open file.
|
|
3451
|
+
* @param {QvdRowWindow} window The rows to read.
|
|
3452
|
+
* @param {boolean} headerOnly Stop once the XML header has been read.
|
|
3453
|
+
* @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
|
|
3454
|
+
* @param {(error: unknown) => never} failed The read's `rethrowAsIoError` handler.
|
|
3455
|
+
* @private
|
|
3456
|
+
*/
|
|
3457
|
+
async _readFrom(handle, window, headerOnly, liveRows, failed) {
|
|
2958
3458
|
const HEADER_DELIMITER = "\r\n\0";
|
|
2959
3459
|
const CHUNK_SIZE = 64 * 1024;
|
|
2960
|
-
const stream = fs.createReadStream(this._path, {
|
|
2961
|
-
highWaterMark: CHUNK_SIZE
|
|
2962
|
-
});
|
|
2963
3460
|
const headerChunks = [];
|
|
2964
3461
|
let headerBytes = 0;
|
|
2965
3462
|
let tail = Buffer.alloc(0);
|
|
2966
3463
|
let headerDelimiterIndex = -1;
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
stream.destroy();
|
|
2977
|
-
break;
|
|
2978
|
-
}
|
|
2979
|
-
tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
|
|
2980
|
-
if (headerBytes > MAX_HEADER_SIZE) {
|
|
2981
|
-
stream.destroy();
|
|
2982
|
-
throw new QvdCorruptedError(
|
|
2983
|
-
`The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
|
|
2984
|
-
{
|
|
2985
|
-
file: this._path,
|
|
2986
|
-
bytesSearched: headerBytes,
|
|
2987
|
-
maxHeaderSize: MAX_HEADER_SIZE,
|
|
2988
|
-
stage: "readData"
|
|
2989
|
-
}
|
|
2990
|
-
);
|
|
2991
|
-
}
|
|
3464
|
+
for await (const chunk of chunksFrom(handle, CHUNK_SIZE, failed)) {
|
|
3465
|
+
const chunkStart = headerBytes;
|
|
3466
|
+
const searchBuffer = tail.length > 0 ? Buffer.concat([tail, chunk]) : chunk;
|
|
3467
|
+
const foundInSearch = searchBuffer.indexOf(HEADER_DELIMITER);
|
|
3468
|
+
headerChunks.push(chunk);
|
|
3469
|
+
headerBytes += chunk.length;
|
|
3470
|
+
if (foundInSearch !== -1) {
|
|
3471
|
+
headerDelimiterIndex = chunkStart - tail.length + foundInSearch;
|
|
3472
|
+
break;
|
|
2992
3473
|
}
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
3474
|
+
tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
|
|
3475
|
+
if (headerBytes > MAX_HEADER_SIZE) {
|
|
3476
|
+
throw new QvdCorruptedError(
|
|
3477
|
+
`The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
|
|
3478
|
+
{
|
|
3479
|
+
file: this._path,
|
|
3480
|
+
bytesSearched: headerBytes,
|
|
3481
|
+
maxHeaderSize: MAX_HEADER_SIZE,
|
|
3482
|
+
stage: "readData"
|
|
3483
|
+
}
|
|
3484
|
+
);
|
|
2998
3485
|
}
|
|
2999
|
-
} finally {
|
|
3000
|
-
await closeReadStream(stream);
|
|
3001
3486
|
}
|
|
3002
3487
|
if (headerDelimiterIndex === -1) {
|
|
3003
3488
|
throw new QvdCorruptedError(
|
|
@@ -3038,9 +3523,9 @@ var init_QvdFileReader = __esm({
|
|
|
3038
3523
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3039
3524
|
);
|
|
3040
3525
|
if (headerNumbersUsable) {
|
|
3041
|
-
const { size:
|
|
3042
|
-
this._fileSize =
|
|
3043
|
-
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <=
|
|
3526
|
+
const { size: fileSize2 } = await handle.stat().catch(failed);
|
|
3527
|
+
this._fileSize = fileSize2;
|
|
3528
|
+
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize2;
|
|
3044
3529
|
}
|
|
3045
3530
|
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3046
3531
|
const windowRows = resolved.limit;
|
|
@@ -3057,7 +3542,7 @@ var init_QvdFileReader = __esm({
|
|
|
3057
3542
|
);
|
|
3058
3543
|
}
|
|
3059
3544
|
if (window.offset === 0 && window.limit === null) {
|
|
3060
|
-
this._buffer = await
|
|
3545
|
+
this._buffer = await handle.readFile().catch(failed);
|
|
3061
3546
|
this._fileSize = this._buffer.length;
|
|
3062
3547
|
this._bufferFirstRow = 0;
|
|
3063
3548
|
this._emitProgress("read", 1, 1);
|
|
@@ -3083,34 +3568,29 @@ var init_QvdFileReader = __esm({
|
|
|
3083
3568
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
3084
3569
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
3085
3570
|
const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
|
|
3086
|
-
const
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
stage: "readData"
|
|
3096
|
-
});
|
|
3097
|
-
}
|
|
3098
|
-
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
3099
|
-
await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3100
|
-
if (indexTableBytesToRead > 0) {
|
|
3101
|
-
await this._readRange(
|
|
3102
|
-
fd,
|
|
3103
|
-
indexTableOffset,
|
|
3104
|
-
indexTableBytesToRead,
|
|
3105
|
-
indexTableOffset + skippedIndexBytes,
|
|
3106
|
-
fileSize,
|
|
3107
|
-
fileBytesRequired
|
|
3108
|
-
);
|
|
3109
|
-
}
|
|
3110
|
-
this._bufferFirstRow = resolved.offset;
|
|
3111
|
-
} finally {
|
|
3112
|
-
await fd.close();
|
|
3571
|
+
const { size: fileSize } = await handle.stat().catch(failed);
|
|
3572
|
+
this._fileSize = fileSize;
|
|
3573
|
+
if (fileBytesRequired > fileSize) {
|
|
3574
|
+
throw new QvdCorruptedError("The file is shorter than its header claims.", {
|
|
3575
|
+
file: this._path,
|
|
3576
|
+
fileSize,
|
|
3577
|
+
requiredBytes: fileBytesRequired,
|
|
3578
|
+
stage: "readData"
|
|
3579
|
+
});
|
|
3113
3580
|
}
|
|
3581
|
+
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
3582
|
+
await this._readRange(handle, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3583
|
+
if (indexTableBytesToRead > 0) {
|
|
3584
|
+
await this._readRange(
|
|
3585
|
+
handle,
|
|
3586
|
+
indexTableOffset,
|
|
3587
|
+
indexTableBytesToRead,
|
|
3588
|
+
indexTableOffset + skippedIndexBytes,
|
|
3589
|
+
fileSize,
|
|
3590
|
+
fileBytesRequired
|
|
3591
|
+
);
|
|
3592
|
+
}
|
|
3593
|
+
this._bufferFirstRow = resolved.offset;
|
|
3114
3594
|
this._emitProgress("read", 1, 1);
|
|
3115
3595
|
}
|
|
3116
3596
|
/**
|
|
@@ -3129,11 +3609,12 @@ var init_QvdFileReader = __esm({
|
|
|
3129
3609
|
* @private
|
|
3130
3610
|
*/
|
|
3131
3611
|
async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
|
|
3132
|
-
|
|
3612
|
+
assert3(this._buffer, "The read buffer has not been allocated.");
|
|
3613
|
+
const failed = rethrowAsIoError(this._path, "read");
|
|
3133
3614
|
let done = 0;
|
|
3134
3615
|
while (done < byteCount) {
|
|
3135
3616
|
const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
|
|
3136
|
-
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
|
|
3617
|
+
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done).catch(failed);
|
|
3137
3618
|
if (bytesRead === 0) {
|
|
3138
3619
|
throw new QvdCorruptedError("Unexpected end of file while reading QVD data.", {
|
|
3139
3620
|
file: this._path,
|
|
@@ -3180,6 +3661,7 @@ var init_QvdFileReader = __esm({
|
|
|
3180
3661
|
}
|
|
3181
3662
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3182
3663
|
const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
|
|
3664
|
+
this._fieldBitMetadataValidated = false;
|
|
3183
3665
|
this._header = await xml2.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
|
|
3184
3666
|
if (!this._header) {
|
|
3185
3667
|
throw new QvdParseError("The XML header could not be parsed.", {
|
|
@@ -3225,8 +3707,8 @@ var init_QvdFileReader = __esm({
|
|
|
3225
3707
|
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
3226
3708
|
* @param {string} stage Stage name for any error raised here.
|
|
3227
3709
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
3228
|
-
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the
|
|
3229
|
-
* record, so the decoder always counts from zero.
|
|
3710
|
+
* indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
|
|
3711
|
+
* window's first record, file row `firstRow`, so the decoder always counts from zero.
|
|
3230
3712
|
* @private
|
|
3231
3713
|
*/
|
|
3232
3714
|
_planIndexTable(window, stage) {
|
|
@@ -3268,11 +3750,11 @@ var init_QvdFileReader = __esm({
|
|
|
3268
3750
|
}
|
|
3269
3751
|
this._fieldBitMetadataValidated = true;
|
|
3270
3752
|
}
|
|
3271
|
-
|
|
3753
|
+
assert3(
|
|
3272
3754
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
3273
3755
|
`The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
|
|
3274
3756
|
);
|
|
3275
|
-
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
|
|
3757
|
+
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
|
|
3276
3758
|
}
|
|
3277
3759
|
/**
|
|
3278
3760
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
@@ -3299,6 +3781,9 @@ var init_QvdFileReader = __esm({
|
|
|
3299
3781
|
const bitOffset = parseInt(field["BitOffset"], 10);
|
|
3300
3782
|
const bitWidth = parseInt(field["BitWidth"], 10);
|
|
3301
3783
|
const bias = parseInt(field["Bias"], 10);
|
|
3784
|
+
const length = parseInt(field["Length"], 10);
|
|
3785
|
+
let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
|
|
3786
|
+
let counted = false;
|
|
3302
3787
|
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3303
3788
|
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3304
3789
|
decodeIndexColumn(
|
|
@@ -3311,15 +3796,53 @@ var init_QvdFileReader = __esm({
|
|
|
3311
3796
|
column
|
|
3312
3797
|
);
|
|
3313
3798
|
for (let row = 0; row < count; row++) {
|
|
3314
|
-
if (column[row] >= 0) {
|
|
3799
|
+
if (column[row] >= 0 && column[row] < indexLimit) {
|
|
3315
3800
|
needed.add(column[row]);
|
|
3316
3801
|
}
|
|
3317
3802
|
}
|
|
3803
|
+
if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
|
|
3804
|
+
counted = true;
|
|
3805
|
+
indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
|
|
3806
|
+
for (const index of needed) {
|
|
3807
|
+
if (index >= indexLimit) {
|
|
3808
|
+
needed.delete(index);
|
|
3809
|
+
}
|
|
3810
|
+
}
|
|
3811
|
+
}
|
|
3318
3812
|
}
|
|
3319
3813
|
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
3320
3814
|
});
|
|
3321
3815
|
return symbolUsage;
|
|
3322
3816
|
}
|
|
3817
|
+
/**
|
|
3818
|
+
* How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
|
|
3819
|
+
*
|
|
3820
|
+
* The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
|
|
3821
|
+
* `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
|
|
3822
|
+
* validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
|
|
3823
|
+
* same way wherever it is met.
|
|
3824
|
+
*
|
|
3825
|
+
* @param {any} field The field's header.
|
|
3826
|
+
* @return {number} Its symbols.
|
|
3827
|
+
* @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
|
|
3828
|
+
* @private
|
|
3829
|
+
*/
|
|
3830
|
+
_countFieldSymbols(field) {
|
|
3831
|
+
assert3(
|
|
3832
|
+
this._buffer && this._symbolTableOffset && this._indexTableOffset,
|
|
3833
|
+
"The QVD file has not been read before its symbols were counted."
|
|
3834
|
+
);
|
|
3835
|
+
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3836
|
+
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3837
|
+
const offset = parseInt(field["Offset"], 10);
|
|
3838
|
+
return countFieldSymbols(
|
|
3839
|
+
symbolBuffer,
|
|
3840
|
+
offset,
|
|
3841
|
+
offset + parseInt(field["Length"], 10),
|
|
3842
|
+
field["FieldName"],
|
|
3843
|
+
this._path
|
|
3844
|
+
);
|
|
3845
|
+
}
|
|
3323
3846
|
/**
|
|
3324
3847
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
3325
3848
|
* and should not be called directly.
|
|
@@ -3405,12 +3928,19 @@ var init_QvdFileReader = __esm({
|
|
|
3405
3928
|
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3406
3929
|
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3407
3930
|
*
|
|
3931
|
+
* Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
|
|
3932
|
+
* NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
|
|
3933
|
+
* are built, because every read decodes through this method and a columnar read hands its codes
|
|
3934
|
+
* straight to the caller. Rows outside the window are not decoded, so they are not checked.
|
|
3935
|
+
*
|
|
3408
3936
|
* @param {QvdRowWindow} window The rows to decode.
|
|
3937
|
+
* @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
|
|
3409
3938
|
*/
|
|
3410
3939
|
async _parseIndexTable(window) {
|
|
3411
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
3412
|
-
this.
|
|
3413
|
-
|
|
3940
|
+
const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
|
|
3941
|
+
assert3(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
3942
|
+
const symbolTable = this._symbolTable;
|
|
3943
|
+
const columns = fields.map((field, position) => {
|
|
3414
3944
|
this._throwIfAborted();
|
|
3415
3945
|
const column = decodeIndexColumn(
|
|
3416
3946
|
indexBuffer,
|
|
@@ -3419,11 +3949,14 @@ var init_QvdFileReader = __esm({
|
|
|
3419
3949
|
parseInt(field["BitOffset"], 10),
|
|
3420
3950
|
parseInt(field["BitWidth"], 10),
|
|
3421
3951
|
parseInt(field["Bias"], 10),
|
|
3422
|
-
new Int32Array(rowsToLoad)
|
|
3952
|
+
new Int32Array(rowsToLoad),
|
|
3953
|
+
{ symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
|
|
3423
3954
|
);
|
|
3424
3955
|
this._emitProgress("index-table", position + 1, fields.length);
|
|
3425
3956
|
return column;
|
|
3426
3957
|
});
|
|
3958
|
+
this._indexColumns = columns;
|
|
3959
|
+
this._rowsDecoded = rowsToLoad;
|
|
3427
3960
|
}
|
|
3428
3961
|
/**
|
|
3429
3962
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -3445,7 +3978,7 @@ var init_QvdFileReader = __esm({
|
|
|
3445
3978
|
await this._parseHeader();
|
|
3446
3979
|
this._emitProgress("header", 1, 1);
|
|
3447
3980
|
this._throwIfAborted();
|
|
3448
|
-
|
|
3981
|
+
assert3(this._header, "The QVD file header has not been parsed.");
|
|
3449
3982
|
const header = this._header["QvdTableHeader"];
|
|
3450
3983
|
let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
|
|
3451
3984
|
if (!Array.isArray(fields)) {
|
|
@@ -3509,9 +4042,10 @@ var init_QvdFileReader = __esm({
|
|
|
3509
4042
|
* Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
|
|
3510
4043
|
* What it keeps instead is what the decoder already produced: one `Int32Array` of stored
|
|
3511
4044
|
* indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
|
|
3512
|
-
* fixture that is
|
|
4045
|
+
* fixture that is 133 MiB against the 352 MiB `data` retains, because a column costs four
|
|
3513
4046
|
* bytes per row rather than a boxed value per cell, and the symbols are a few thousand
|
|
3514
|
-
* entries shared across every row that uses them.
|
|
4047
|
+
* entries shared across every row that uses them. The codes' storage lives outside the V8
|
|
4048
|
+
* heap: 3 MiB of the 133 is on it.
|
|
3515
4049
|
*
|
|
3516
4050
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3517
4051
|
* The rows to decode, in the same spellings `load()` accepts.
|
|
@@ -3522,7 +4056,7 @@ var init_QvdFileReader = __esm({
|
|
|
3522
4056
|
const prepared = await this._prepare(rows, null, true);
|
|
3523
4057
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3524
4058
|
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
3525
|
-
|
|
4059
|
+
assert3(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
3526
4060
|
return new QvdColumnTable2({
|
|
3527
4061
|
columns: prepared.columns,
|
|
3528
4062
|
codesByField: this._indexColumns,
|
|
@@ -3551,7 +4085,8 @@ var init_QvdFileReader = __esm({
|
|
|
3551
4085
|
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
3552
4086
|
*
|
|
3553
4087
|
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
3554
|
-
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
4088
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
|
|
4089
|
+
* is still checked, as every read of a file's rows checks it.
|
|
3555
4090
|
*
|
|
3556
4091
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
3557
4092
|
* The rows to cover, in the same spellings `load()` accepts.
|
|
@@ -3569,6 +4104,10 @@ var init_QvdFileReader = __esm({
|
|
|
3569
4104
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
3570
4105
|
const rows = normaliseWindow(window, this._path);
|
|
3571
4106
|
const prepared = await this._prepare(rows, liveRows);
|
|
4107
|
+
if (prepared.rowsAvailable === 0) {
|
|
4108
|
+
this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
|
|
4109
|
+
return;
|
|
4110
|
+
}
|
|
3572
4111
|
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
3573
4112
|
this._throwIfAborted();
|
|
3574
4113
|
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
@@ -3619,7 +4158,7 @@ var init_QvdFileReader = __esm({
|
|
|
3619
4158
|
await this._parseHeader();
|
|
3620
4159
|
this._emitProgress("header", 1, 1);
|
|
3621
4160
|
this._throwIfAborted();
|
|
3622
|
-
|
|
4161
|
+
assert3(this._header, "The QVD file header has not been parsed.");
|
|
3623
4162
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
3624
4163
|
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3625
4164
|
const resolved = resolveWindow(window, totalRows);
|
|
@@ -3633,9 +4172,9 @@ var init_QvdFileReader = __esm({
|
|
|
3633
4172
|
}
|
|
3634
4173
|
}
|
|
3635
4174
|
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
3636
|
-
|
|
4175
|
+
assert3(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
3637
4176
|
this._throwIfAborted();
|
|
3638
|
-
|
|
4177
|
+
assert3(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3639
4178
|
const resolvedByField = [];
|
|
3640
4179
|
const halvesByField = [];
|
|
3641
4180
|
const entries = [];
|
|
@@ -3696,7 +4235,7 @@ var init_QvdFileReader = __esm({
|
|
|
3696
4235
|
* @private
|
|
3697
4236
|
*/
|
|
3698
4237
|
_buildRows(resolvedByField, progressBase, progressTotal) {
|
|
3699
|
-
|
|
4238
|
+
assert3(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
3700
4239
|
const indexColumns = this._indexColumns;
|
|
3701
4240
|
const fieldCount = indexColumns.length;
|
|
3702
4241
|
const rowCount = this._rowsDecoded;
|
|
@@ -4301,14 +4840,31 @@ var init_QvdDataFrame = __esm({
|
|
|
4301
4840
|
* volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
|
|
4302
4841
|
* back to the working directory rather than removing the restriction.
|
|
4303
4842
|
* @param {Function} [options.onProgress] Optional progress callback function that receives progress updates during write operations.
|
|
4843
|
+
* @param {boolean} [options.atomic=true] Whether to replace the file rather than rewrite it. The QVD
|
|
4844
|
+
* is built beside it under a temporary name and renamed over it, so a write that fails leaves the
|
|
4845
|
+
* previous file exactly as it was, and a reader of the path - a Qlik reload, say - sees the old
|
|
4846
|
+
* file or the new one and never a part of either. False rewrites the file where it stands, as
|
|
4847
|
+
* every version before 2.0.0 did: that keeps its identity, including hard links, its owner and
|
|
4848
|
+
* permissions set on the file itself, and needs neither room for two copies nor permission to
|
|
4849
|
+
* create files in the directory. It also destroys the previous file as the write begins. A file
|
|
4850
|
+
* that does not exist yet is renamed into place either way, since there is nothing to rewrite.
|
|
4851
|
+
* @param {boolean} [options.fsync=true] Whether to wait for the contents to reach the disk before
|
|
4852
|
+
* resolving. It is what carries an atomic write's promise through a power loss - the contents are
|
|
4853
|
+
* on the disk before anything is renamed, so a crash leaves the previous file or the new one and
|
|
4854
|
+
* never a damaged one - and what reports a failing disk's deferred error instead of losing it. The
|
|
4855
|
+
* rename itself is not flushed, so a crash just after the call can still lose the replacement, or
|
|
4856
|
+
* a file that did not exist before. False resolves once the operating system has accepted the
|
|
4857
|
+
* bytes, which is faster and is what every version before 2.0.0 did.
|
|
4304
4858
|
*/
|
|
4305
|
-
async toQvd(
|
|
4859
|
+
async toQvd(path5, options = {}) {
|
|
4306
4860
|
const { QvdFileWriter: QvdFileWriter2 } = await Promise.resolve().then(() => (init_QvdFileWriter(), QvdFileWriter_exports));
|
|
4307
4861
|
const writerOptions = {
|
|
4308
4862
|
allowedDir: options.allowedDir,
|
|
4309
|
-
onProgress: options.onProgress
|
|
4863
|
+
onProgress: options.onProgress,
|
|
4864
|
+
atomic: options.atomic,
|
|
4865
|
+
fsync: options.fsync
|
|
4310
4866
|
};
|
|
4311
|
-
await new QvdFileWriter2(
|
|
4867
|
+
await new QvdFileWriter2(path5, this, writerOptions).save();
|
|
4312
4868
|
}
|
|
4313
4869
|
/**
|
|
4314
4870
|
* Loads a QVD file and returns its data frame.
|
|
@@ -4361,9 +4917,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4361
4917
|
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
4362
4918
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
4363
4919
|
*/
|
|
4364
|
-
static async fromQvd(
|
|
4920
|
+
static async fromQvd(path5, options = {}) {
|
|
4365
4921
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4366
|
-
return await new QvdFileReader2(
|
|
4922
|
+
return await new QvdFileReader2(path5, readerOptionsFrom(options)).load(windowFrom(options));
|
|
4367
4923
|
}
|
|
4368
4924
|
/**
|
|
4369
4925
|
* Reads a QVD file in chunks, as an async generator of data frames.
|
|
@@ -4410,9 +4966,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4410
4966
|
* windowed read switches to two-pass filtering.
|
|
4411
4967
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
4412
4968
|
*/
|
|
4413
|
-
static async *iterate(
|
|
4969
|
+
static async *iterate(path5, options = {}) {
|
|
4414
4970
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4415
|
-
const reader = new QvdFileReader2(
|
|
4971
|
+
const reader = new QvdFileReader2(path5, readerOptionsFrom(options));
|
|
4416
4972
|
yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
|
|
4417
4973
|
}
|
|
4418
4974
|
/**
|
|
@@ -4443,9 +4999,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4443
4999
|
* @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
|
|
4444
5000
|
* @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
|
|
4445
5001
|
*/
|
|
4446
|
-
static async readMetadata(
|
|
5002
|
+
static async readMetadata(path5, options = {}) {
|
|
4447
5003
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4448
|
-
return await new QvdFileReader2(
|
|
5004
|
+
return await new QvdFileReader2(path5, metadataOptionsFrom(options)).loadMetadata();
|
|
4449
5005
|
}
|
|
4450
5006
|
/**
|
|
4451
5007
|
* Constructs a data frame from a dictionary.
|