qvdjs 1.0.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -1
- package/README.md +102 -32
- package/dist/index.cjs +745 -189
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +741 -185
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
package/dist/index.cjs
CHANGED
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
4
|
-
var
|
|
5
|
-
var
|
|
3
|
+
var fs2 = require('fs');
|
|
4
|
+
var path2 = require('path');
|
|
5
|
+
var assert3 = require('assert');
|
|
6
|
+
var crypto2 = require('crypto');
|
|
7
|
+
var promises = require('timers/promises');
|
|
6
8
|
var xml2 = require('xml2js');
|
|
7
|
-
var assert2 = require('assert');
|
|
8
9
|
var os = require('os');
|
|
9
10
|
var v8 = require('v8');
|
|
10
11
|
var util = require('util');
|
|
11
12
|
|
|
12
13
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
13
14
|
|
|
14
|
-
var
|
|
15
|
-
var
|
|
16
|
-
var
|
|
15
|
+
var fs2__default = /*#__PURE__*/_interopDefault(fs2);
|
|
16
|
+
var path2__default = /*#__PURE__*/_interopDefault(path2);
|
|
17
|
+
var assert3__default = /*#__PURE__*/_interopDefault(assert3);
|
|
18
|
+
var crypto2__default = /*#__PURE__*/_interopDefault(crypto2);
|
|
17
19
|
var xml2__default = /*#__PURE__*/_interopDefault(xml2);
|
|
18
|
-
var assert2__default = /*#__PURE__*/_interopDefault(assert2);
|
|
19
20
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
20
21
|
var v8__default = /*#__PURE__*/_interopDefault(v8);
|
|
21
22
|
|
|
@@ -45,9 +46,11 @@ var init_QvdErrors = __esm({
|
|
|
45
46
|
* @param {string} message The error message.
|
|
46
47
|
* @param {string} code The error code.
|
|
47
48
|
* @param {Object} [context={}] Additional context about the error.
|
|
49
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`, so a `cause` becomes `error.cause`: the
|
|
50
|
+
* error this one reports, such as the operating system's refusal behind a `QvdIOError`.
|
|
48
51
|
*/
|
|
49
|
-
constructor(message, code, context = {}) {
|
|
50
|
-
super(message);
|
|
52
|
+
constructor(message, code, context = {}, options = void 0) {
|
|
53
|
+
super(message, options);
|
|
51
54
|
this.name = ERROR_NAMES.get(new.target) ?? new.target.name;
|
|
52
55
|
this.code = code;
|
|
53
56
|
this.context = context;
|
|
@@ -63,9 +66,10 @@ var init_QvdErrors = __esm({
|
|
|
63
66
|
*
|
|
64
67
|
* @param {string} message The error message.
|
|
65
68
|
* @param {Object} [context={}] Additional context about the error.
|
|
69
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
66
70
|
*/
|
|
67
|
-
constructor(message, context = {}) {
|
|
68
|
-
super(message, "QVD_PARSE_ERROR", context);
|
|
71
|
+
constructor(message, context = {}, options = void 0) {
|
|
72
|
+
super(message, "QVD_PARSE_ERROR", context, options);
|
|
69
73
|
}
|
|
70
74
|
};
|
|
71
75
|
exports.QvdValidationError = class extends exports.QvdError {
|
|
@@ -77,9 +81,10 @@ var init_QvdErrors = __esm({
|
|
|
77
81
|
*
|
|
78
82
|
* @param {string} message The error message.
|
|
79
83
|
* @param {Object} [context={}] Additional context about the error.
|
|
84
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
80
85
|
*/
|
|
81
|
-
constructor(message, context = {}) {
|
|
82
|
-
super(message, "QVD_VALIDATION_ERROR", context);
|
|
86
|
+
constructor(message, context = {}, options = void 0) {
|
|
87
|
+
super(message, "QVD_VALIDATION_ERROR", context, options);
|
|
83
88
|
}
|
|
84
89
|
};
|
|
85
90
|
exports.QvdIOError = class extends exports.QvdError {
|
|
@@ -91,9 +96,10 @@ var init_QvdErrors = __esm({
|
|
|
91
96
|
*
|
|
92
97
|
* @param {string} message The error message.
|
|
93
98
|
* @param {Object} [context={}] Additional context about the error.
|
|
99
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
94
100
|
*/
|
|
95
|
-
constructor(message, context = {}) {
|
|
96
|
-
super(message, "QVD_IO_ERROR", context);
|
|
101
|
+
constructor(message, context = {}, options = void 0) {
|
|
102
|
+
super(message, "QVD_IO_ERROR", context, options);
|
|
97
103
|
}
|
|
98
104
|
};
|
|
99
105
|
exports.QvdCorruptedError = class extends exports.QvdError {
|
|
@@ -105,9 +111,10 @@ var init_QvdErrors = __esm({
|
|
|
105
111
|
*
|
|
106
112
|
* @param {string} message The error message.
|
|
107
113
|
* @param {Object} [context={}] Additional context about the error.
|
|
114
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
108
115
|
*/
|
|
109
|
-
constructor(message, context = {}) {
|
|
110
|
-
super(message, "QVD_CORRUPTED_ERROR", context);
|
|
116
|
+
constructor(message, context = {}, options = void 0) {
|
|
117
|
+
super(message, "QVD_CORRUPTED_ERROR", context, options);
|
|
111
118
|
}
|
|
112
119
|
};
|
|
113
120
|
exports.QvdSecurityError = class extends exports.QvdError {
|
|
@@ -119,9 +126,10 @@ var init_QvdErrors = __esm({
|
|
|
119
126
|
*
|
|
120
127
|
* @param {string} message The error message.
|
|
121
128
|
* @param {Object} [context={}] Additional context about the error.
|
|
129
|
+
* @param {{cause?: unknown}} [options] Passed on to `Error`; a `cause` becomes `error.cause`.
|
|
122
130
|
*/
|
|
123
|
-
constructor(message, context = {}) {
|
|
124
|
-
super(message, "QVD_SECURITY_ERROR", context);
|
|
131
|
+
constructor(message, context = {}, options = void 0) {
|
|
132
|
+
super(message, "QVD_SECURITY_ERROR", context, options);
|
|
125
133
|
}
|
|
126
134
|
};
|
|
127
135
|
ERROR_NAMES.set(exports.QvdError, "QvdError");
|
|
@@ -419,6 +427,28 @@ var init_QvdDual = __esm({
|
|
|
419
427
|
}
|
|
420
428
|
});
|
|
421
429
|
|
|
430
|
+
// src/util/optionTypes.js
|
|
431
|
+
function booleanOption(value, { option, file, whenUnset }) {
|
|
432
|
+
if (value === void 0 || value === null) {
|
|
433
|
+
return whenUnset;
|
|
434
|
+
}
|
|
435
|
+
if (typeof value !== "boolean") {
|
|
436
|
+
throw new exports.QvdValidationError(`${option} must be true or false`, {
|
|
437
|
+
option,
|
|
438
|
+
provided: value,
|
|
439
|
+
type: typeof value,
|
|
440
|
+
file
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
return value;
|
|
444
|
+
}
|
|
445
|
+
var init_optionTypes = __esm({
|
|
446
|
+
"src/util/optionTypes.js"() {
|
|
447
|
+
init_QvdErrors();
|
|
448
|
+
__name(booleanOption, "booleanOption");
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
|
|
422
452
|
// src/util/readOptions.js
|
|
423
453
|
function requireRowCount(value, name, filePath) {
|
|
424
454
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
@@ -529,18 +559,7 @@ function normaliseDuals(value, filePath) {
|
|
|
529
559
|
return value;
|
|
530
560
|
}
|
|
531
561
|
function normaliseCoerceNumericStrings(value, filePath) {
|
|
532
|
-
|
|
533
|
-
return false;
|
|
534
|
-
}
|
|
535
|
-
if (typeof value !== "boolean") {
|
|
536
|
-
throw new exports.QvdValidationError("coerceNumericStrings must be true or false", {
|
|
537
|
-
option: "coerceNumericStrings",
|
|
538
|
-
provided: value,
|
|
539
|
-
type: typeof value,
|
|
540
|
-
file: filePath
|
|
541
|
-
});
|
|
542
|
-
}
|
|
543
|
-
return value;
|
|
562
|
+
return booleanOption(value, { option: "coerceNumericStrings", file: filePath, whenUnset: false });
|
|
544
563
|
}
|
|
545
564
|
function readerOptionsFrom(options) {
|
|
546
565
|
return {
|
|
@@ -568,6 +587,7 @@ var DUAL_MODES;
|
|
|
568
587
|
var init_readOptions = __esm({
|
|
569
588
|
"src/util/readOptions.js"() {
|
|
570
589
|
init_QvdErrors();
|
|
590
|
+
init_optionTypes();
|
|
571
591
|
__name(requireRowCount, "requireRowCount");
|
|
572
592
|
__name(normaliseWindow, "normaliseWindow");
|
|
573
593
|
__name(resolveWindow, "resolveWindow");
|
|
@@ -711,24 +731,64 @@ var init_storedSymbols = __esm({
|
|
|
711
731
|
__name(sameValueZero, "sameValueZero");
|
|
712
732
|
}
|
|
713
733
|
});
|
|
734
|
+
function readDanglingLink(target) {
|
|
735
|
+
let link;
|
|
736
|
+
try {
|
|
737
|
+
link = fs2__default.default.lstatSync(target);
|
|
738
|
+
} catch {
|
|
739
|
+
return NOT_A_LINK;
|
|
740
|
+
}
|
|
741
|
+
if (!link.isSymbolicLink()) {
|
|
742
|
+
return NOT_A_LINK;
|
|
743
|
+
}
|
|
744
|
+
let destination;
|
|
745
|
+
try {
|
|
746
|
+
destination = fs2__default.default.readlinkSync(target);
|
|
747
|
+
} catch {
|
|
748
|
+
return UNREADABLE_LINK;
|
|
749
|
+
}
|
|
750
|
+
if (path2__default.default.isAbsolute(destination)) {
|
|
751
|
+
return destination;
|
|
752
|
+
}
|
|
753
|
+
try {
|
|
754
|
+
return path2__default.default.resolve(fs2__default.default.realpathSync(path2__default.default.dirname(target)), destination);
|
|
755
|
+
} catch {
|
|
756
|
+
return UNREADABLE_LINK;
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
var MAX_LINK_HOPS, NOT_A_LINK, UNREADABLE_LINK;
|
|
760
|
+
var init_linkTarget = __esm({
|
|
761
|
+
"src/util/linkTarget.js"() {
|
|
762
|
+
MAX_LINK_HOPS = 40;
|
|
763
|
+
NOT_A_LINK = /* @__PURE__ */ Symbol("not a link");
|
|
764
|
+
UNREADABLE_LINK = /* @__PURE__ */ Symbol("unreadable link");
|
|
765
|
+
__name(readDanglingLink, "readDanglingLink");
|
|
766
|
+
}
|
|
767
|
+
});
|
|
714
768
|
function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
|
|
715
769
|
const isCaseInsensitiveFS = process.platform === "win32";
|
|
716
770
|
const base = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
|
|
717
771
|
const target = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
|
|
718
|
-
const relative =
|
|
772
|
+
const relative = path2__default.default.relative(base, target);
|
|
719
773
|
if (relative === "") {
|
|
720
774
|
return true;
|
|
721
775
|
}
|
|
722
|
-
if (
|
|
776
|
+
if (path2__default.default.isAbsolute(relative)) {
|
|
723
777
|
return false;
|
|
724
778
|
}
|
|
725
|
-
return relative !== ".." && !relative.startsWith(`..${
|
|
779
|
+
return relative !== ".." && !relative.startsWith(`..${path2__default.default.sep}`);
|
|
726
780
|
}
|
|
727
781
|
function resolveDeepestExisting(target) {
|
|
728
782
|
let current = target;
|
|
783
|
+
let farEnd = target;
|
|
784
|
+
let walkedUp = false;
|
|
785
|
+
let followedWhileClimbing = false;
|
|
786
|
+
let hops = 0;
|
|
729
787
|
for (; ; ) {
|
|
730
788
|
try {
|
|
731
|
-
|
|
789
|
+
const deepest = fs2__default.default.realpathSync(current);
|
|
790
|
+
const canonicalFarEnd = walkedUp && !followedWhileClimbing ? path2__default.default.join(deepest, path2__default.default.relative(current, farEnd)) : farEnd;
|
|
791
|
+
return { deepest, exists: !walkedUp, farEnd: canonicalFarEnd };
|
|
732
792
|
} catch (error) {
|
|
733
793
|
const code = (
|
|
734
794
|
/** @type {{code?: string}} */
|
|
@@ -737,10 +797,27 @@ function resolveDeepestExisting(target) {
|
|
|
737
797
|
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
738
798
|
return null;
|
|
739
799
|
}
|
|
740
|
-
const
|
|
800
|
+
const followed = readDanglingLink(current);
|
|
801
|
+
if (followed === UNREADABLE_LINK) {
|
|
802
|
+
return REFUSED;
|
|
803
|
+
}
|
|
804
|
+
if (followed !== NOT_A_LINK) {
|
|
805
|
+
if (++hops > MAX_LINK_HOPS) {
|
|
806
|
+
return REFUSED;
|
|
807
|
+
}
|
|
808
|
+
current = followed;
|
|
809
|
+
if (walkedUp) {
|
|
810
|
+
followedWhileClimbing = true;
|
|
811
|
+
} else {
|
|
812
|
+
farEnd = followed;
|
|
813
|
+
}
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
816
|
+
const parent = path2__default.default.dirname(current);
|
|
741
817
|
if (parent === current) {
|
|
742
818
|
return null;
|
|
743
819
|
}
|
|
820
|
+
walkedUp = true;
|
|
744
821
|
current = parent;
|
|
745
822
|
}
|
|
746
823
|
}
|
|
@@ -748,32 +825,42 @@ function resolveDeepestExisting(target) {
|
|
|
748
825
|
function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
|
|
749
826
|
let baseStat;
|
|
750
827
|
try {
|
|
751
|
-
baseStat =
|
|
828
|
+
baseStat = fs2__default.default.statSync(fs2__default.default.realpathSync(resolvedBaseDir), { bigint: true });
|
|
752
829
|
} catch {
|
|
753
830
|
return null;
|
|
754
831
|
}
|
|
755
|
-
|
|
756
|
-
if (
|
|
832
|
+
const found = resolveDeepestExisting(resolvedPath);
|
|
833
|
+
if (found === REFUSED) {
|
|
834
|
+
return { contained: false, target: resolvedPath, stats: null };
|
|
835
|
+
}
|
|
836
|
+
if (found === null) {
|
|
757
837
|
return null;
|
|
758
838
|
}
|
|
759
|
-
|
|
839
|
+
const target = found.exists ? found.deepest : found.farEnd;
|
|
840
|
+
let stats = null;
|
|
841
|
+
let current = found.deepest;
|
|
842
|
+
for (let first = true; ; first = false) {
|
|
843
|
+
const ownIdentity = first && found.exists;
|
|
760
844
|
let stat;
|
|
761
845
|
try {
|
|
762
|
-
stat =
|
|
846
|
+
stat = ownIdentity ? fs2__default.default.lstatSync(current, { bigint: true }) : fs2__default.default.statSync(current, { bigint: true });
|
|
763
847
|
} catch {
|
|
764
848
|
return null;
|
|
765
849
|
}
|
|
850
|
+
if (ownIdentity) {
|
|
851
|
+
stats = stat;
|
|
852
|
+
}
|
|
766
853
|
if (stat.dev === baseStat.dev && stat.ino === baseStat.ino) {
|
|
767
|
-
return true;
|
|
854
|
+
return { contained: true, target, stats };
|
|
768
855
|
}
|
|
769
|
-
const parent =
|
|
856
|
+
const parent = path2__default.default.dirname(current);
|
|
770
857
|
if (parent === current) {
|
|
771
|
-
return false;
|
|
858
|
+
return { contained: false, target, stats };
|
|
772
859
|
}
|
|
773
860
|
current = parent;
|
|
774
861
|
}
|
|
775
862
|
}
|
|
776
|
-
function
|
|
863
|
+
function checkPath(filePath, allowedDir) {
|
|
777
864
|
if (typeof filePath !== "string" || filePath.length === 0) {
|
|
778
865
|
throw new exports.QvdValidationError("filePath must be a non-empty string", {
|
|
779
866
|
provided: filePath,
|
|
@@ -792,11 +879,11 @@ function validatePath(filePath, allowedDir) {
|
|
|
792
879
|
reason: "null_byte"
|
|
793
880
|
});
|
|
794
881
|
}
|
|
795
|
-
const resolvedPath =
|
|
882
|
+
const resolvedPath = path2__default.default.resolve(filePath);
|
|
796
883
|
const baseDir = allowedDir || process.cwd();
|
|
797
|
-
const resolvedBaseDir =
|
|
884
|
+
const resolvedBaseDir = path2__default.default.resolve(baseDir);
|
|
798
885
|
const onDisk = isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath);
|
|
799
|
-
const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk;
|
|
886
|
+
const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk.contained;
|
|
800
887
|
if (!contained) {
|
|
801
888
|
throw new exports.QvdSecurityError("Path traversal detected: Access denied", {
|
|
802
889
|
path: filePath,
|
|
@@ -808,15 +895,197 @@ function validatePath(filePath, allowedDir) {
|
|
|
808
895
|
check: onDisk === null ? "lexical" : "filesystem"
|
|
809
896
|
});
|
|
810
897
|
}
|
|
811
|
-
|
|
898
|
+
if (onDisk === null) {
|
|
899
|
+
return { path: resolvedPath, base: resolvedBaseDir, target: resolvedPath, stats: null, onDisk: false };
|
|
900
|
+
}
|
|
901
|
+
return { path: resolvedPath, base: resolvedBaseDir, target: onDisk.target, stats: onDisk.stats, onDisk: true };
|
|
812
902
|
}
|
|
903
|
+
var REFUSED;
|
|
813
904
|
var init_validatePath = __esm({
|
|
814
905
|
"src/util/validatePath.js"() {
|
|
815
906
|
init_QvdErrors();
|
|
907
|
+
init_linkTarget();
|
|
816
908
|
__name(isWithinDirectoryLexically, "isWithinDirectoryLexically");
|
|
909
|
+
REFUSED = /* @__PURE__ */ Symbol("refused");
|
|
817
910
|
__name(resolveDeepestExisting, "resolveDeepestExisting");
|
|
818
911
|
__name(isWithinDirectoryOnDisk, "isWithinDirectoryOnDisk");
|
|
819
|
-
__name(
|
|
912
|
+
__name(checkPath, "checkPath");
|
|
913
|
+
}
|
|
914
|
+
});
|
|
915
|
+
|
|
916
|
+
// src/util/ioErrors.js
|
|
917
|
+
function isSystemError(error) {
|
|
918
|
+
const candidate = (
|
|
919
|
+
/** @type {{code?: unknown, syscall?: unknown}|null} */
|
|
920
|
+
error
|
|
921
|
+
);
|
|
922
|
+
return candidate !== null && typeof candidate === "object" && typeof candidate.syscall === "string" && typeof candidate.code === "string";
|
|
923
|
+
}
|
|
924
|
+
function asIoError(error, file, direction) {
|
|
925
|
+
if (!isSystemError(error)) {
|
|
926
|
+
return error;
|
|
927
|
+
}
|
|
928
|
+
const { message, syscall, code } = (
|
|
929
|
+
/** @type {{message: string, syscall: string, code: string}} */
|
|
930
|
+
error
|
|
931
|
+
);
|
|
932
|
+
return new exports.QvdIOError(
|
|
933
|
+
`Could not ${direction} the QVD file: ${message}`,
|
|
934
|
+
{ file, operation: syscall, code },
|
|
935
|
+
{ cause: error }
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
function rethrowAsIoError(file, direction) {
|
|
939
|
+
return (error) => {
|
|
940
|
+
throw asIoError(error, file, direction);
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
async function closeAfter(handle, failed, use) {
|
|
944
|
+
let result;
|
|
945
|
+
try {
|
|
946
|
+
result = await use();
|
|
947
|
+
} catch (error) {
|
|
948
|
+
await handle.close().catch(() => {
|
|
949
|
+
});
|
|
950
|
+
throw error;
|
|
951
|
+
}
|
|
952
|
+
await handle.close().catch(failed);
|
|
953
|
+
return result;
|
|
954
|
+
}
|
|
955
|
+
var init_ioErrors = __esm({
|
|
956
|
+
"src/util/ioErrors.js"() {
|
|
957
|
+
init_QvdErrors();
|
|
958
|
+
__name(isSystemError, "isSystemError");
|
|
959
|
+
__name(asIoError, "asIoError");
|
|
960
|
+
__name(rethrowAsIoError, "rethrowAsIoError");
|
|
961
|
+
__name(closeAfter, "closeAfter");
|
|
962
|
+
}
|
|
963
|
+
});
|
|
964
|
+
function changedAfterCheck(checked, change) {
|
|
965
|
+
return new exports.QvdSecurityError(`Path traversal detected: ${DESCRIPTIONS[change]}`, {
|
|
966
|
+
path: checked.path,
|
|
967
|
+
resolvedPath: checked.target,
|
|
968
|
+
allowedDir: checked.base,
|
|
969
|
+
reason: "changed_after_check",
|
|
970
|
+
change
|
|
971
|
+
});
|
|
972
|
+
}
|
|
973
|
+
async function openChecked(checked, purpose, failed, { nofollow = NOFOLLOW } = {}) {
|
|
974
|
+
assert3__default.default(purpose === "read" || checked.stats !== null, "A rewrite in place is of a file that exists.");
|
|
975
|
+
const noFollow = checked.onDisk ? nofollow : 0;
|
|
976
|
+
const flags = (purpose === "rewrite" ? O_WRONLY : O_RDONLY) | noFollow;
|
|
977
|
+
let handle;
|
|
978
|
+
try {
|
|
979
|
+
handle = await fs2__default.default.promises.open(checked.target, flags);
|
|
980
|
+
} catch (error) {
|
|
981
|
+
const { code } = (
|
|
982
|
+
/** @type {{code?: string}} */
|
|
983
|
+
error ?? {}
|
|
984
|
+
);
|
|
985
|
+
if (checked.onDisk && noFollow !== 0 && code !== void 0 && BECAME_A_LINK.has(code)) {
|
|
986
|
+
throw changedAfterCheck(checked, "became_a_symlink");
|
|
987
|
+
}
|
|
988
|
+
return failed(error);
|
|
989
|
+
}
|
|
990
|
+
const stats = checked.stats;
|
|
991
|
+
if (stats === null) {
|
|
992
|
+
if (checked.onDisk) {
|
|
993
|
+
await handle.close().catch(() => {
|
|
994
|
+
});
|
|
995
|
+
throw changedAfterCheck(checked, "appeared");
|
|
996
|
+
}
|
|
997
|
+
return handle;
|
|
998
|
+
}
|
|
999
|
+
try {
|
|
1000
|
+
const opened = await handle.stat({ bigint: true }).catch(failed);
|
|
1001
|
+
if (opened.dev !== stats.dev || opened.ino !== stats.ino) {
|
|
1002
|
+
throw changedAfterCheck(checked, "replaced");
|
|
1003
|
+
}
|
|
1004
|
+
if (purpose === "rewrite" && stats.isFile()) {
|
|
1005
|
+
await handle.truncate(0).catch(failed);
|
|
1006
|
+
}
|
|
1007
|
+
} catch (error) {
|
|
1008
|
+
await handle.close().catch(() => {
|
|
1009
|
+
});
|
|
1010
|
+
throw error;
|
|
1011
|
+
}
|
|
1012
|
+
return handle;
|
|
1013
|
+
}
|
|
1014
|
+
var NOFOLLOW, O_RDONLY, O_WRONLY, BECAME_A_LINK, DESCRIPTIONS;
|
|
1015
|
+
var init_openChecked = __esm({
|
|
1016
|
+
"src/util/openChecked.js"() {
|
|
1017
|
+
init_QvdErrors();
|
|
1018
|
+
NOFOLLOW = fs2__default.default.constants.O_NOFOLLOW ?? 0;
|
|
1019
|
+
({ O_RDONLY, O_WRONLY } = fs2__default.default.constants);
|
|
1020
|
+
BECAME_A_LINK = /* @__PURE__ */ new Set(["ELOOP", "EMLINK"]);
|
|
1021
|
+
DESCRIPTIONS = {
|
|
1022
|
+
became_a_symlink: "the file became a symbolic link after it was checked",
|
|
1023
|
+
replaced: "the file was replaced by a different one after it was checked",
|
|
1024
|
+
appeared: "a file appeared at the path after it was checked"
|
|
1025
|
+
};
|
|
1026
|
+
__name(changedAfterCheck, "changedAfterCheck");
|
|
1027
|
+
__name(openChecked, "openChecked");
|
|
1028
|
+
}
|
|
1029
|
+
});
|
|
1030
|
+
function firstBytesOf(name, budget) {
|
|
1031
|
+
if (Buffer.byteLength(name) <= budget) {
|
|
1032
|
+
return name;
|
|
1033
|
+
}
|
|
1034
|
+
let kept = "";
|
|
1035
|
+
let bytes = 0;
|
|
1036
|
+
for (const character of name) {
|
|
1037
|
+
const size = Buffer.byteLength(character);
|
|
1038
|
+
if (bytes + size > budget) {
|
|
1039
|
+
break;
|
|
1040
|
+
}
|
|
1041
|
+
kept += character;
|
|
1042
|
+
bytes += size;
|
|
1043
|
+
}
|
|
1044
|
+
return kept;
|
|
1045
|
+
}
|
|
1046
|
+
function temporaryPathFor(target) {
|
|
1047
|
+
const marks = `.qvdjs-${crypto2__default.default.randomBytes(8).toString("hex")}.tmp`;
|
|
1048
|
+
const name = `${firstBytesOf(path2__default.default.basename(target), MAX_NAME_BYTES - marks.length)}${marks}`;
|
|
1049
|
+
return path2__default.default.join(path2__default.default.dirname(target), name);
|
|
1050
|
+
}
|
|
1051
|
+
async function retrying(call, { platform = process.platform, delays = RETRY_DELAYS } = {}) {
|
|
1052
|
+
for (let attempt = 0; ; attempt++) {
|
|
1053
|
+
try {
|
|
1054
|
+
return await call();
|
|
1055
|
+
} catch (error) {
|
|
1056
|
+
const { code } = (
|
|
1057
|
+
/** @type {{code?: string}} */
|
|
1058
|
+
error ?? {}
|
|
1059
|
+
);
|
|
1060
|
+
if (platform !== "win32" || attempt >= delays.length || code === void 0 || !IN_USE.has(code)) {
|
|
1061
|
+
throw error;
|
|
1062
|
+
}
|
|
1063
|
+
await promises.setTimeout(delays[attempt]);
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
function renameOver(from, to, options) {
|
|
1068
|
+
return retrying(() => fs2__default.default.promises.rename(from, to), options);
|
|
1069
|
+
}
|
|
1070
|
+
async function removeTemporary(file, options) {
|
|
1071
|
+
try {
|
|
1072
|
+
await retrying(() => fs2__default.default.promises.rm(file, { force: true }), options);
|
|
1073
|
+
return true;
|
|
1074
|
+
} catch {
|
|
1075
|
+
return false;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
var MAX_NAME_BYTES, IN_USE, RETRY_DELAYS;
|
|
1079
|
+
var init_replaceFile = __esm({
|
|
1080
|
+
"src/util/replaceFile.js"() {
|
|
1081
|
+
MAX_NAME_BYTES = 255;
|
|
1082
|
+
__name(firstBytesOf, "firstBytesOf");
|
|
1083
|
+
__name(temporaryPathFor, "temporaryPathFor");
|
|
1084
|
+
IN_USE = /* @__PURE__ */ new Set(["EPERM", "EACCES", "EBUSY"]);
|
|
1085
|
+
RETRY_DELAYS = [10, 20, 40, 80, 160, 320];
|
|
1086
|
+
__name(retrying, "retrying");
|
|
1087
|
+
__name(renameOver, "renameOver");
|
|
1088
|
+
__name(removeTemporary, "removeTemporary");
|
|
820
1089
|
}
|
|
821
1090
|
});
|
|
822
1091
|
|
|
@@ -829,8 +1098,12 @@ function fieldGeometry(bitOffset, bitWidth) {
|
|
|
829
1098
|
byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
|
|
830
1099
|
};
|
|
831
1100
|
}
|
|
832
|
-
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
|
|
1101
|
+
function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out, bounds = null) {
|
|
1102
|
+
const symbolCount = bounds === null ? Infinity : bounds.symbolCount;
|
|
833
1103
|
if (bitWidth === 0) {
|
|
1104
|
+
if (bounds !== null && rowCount > 0 && bias >= symbolCount) {
|
|
1105
|
+
refuse2(bounds, 0, bias, bias);
|
|
1106
|
+
}
|
|
834
1107
|
out.fill(bias, 0, rowCount);
|
|
835
1108
|
return out;
|
|
836
1109
|
}
|
|
@@ -844,10 +1117,25 @@ function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bi
|
|
|
844
1117
|
if (byteCount > 2) acc += buffer[base + 2] * 65536;
|
|
845
1118
|
if (byteCount > 3) acc += buffer[base + 3] * 16777216;
|
|
846
1119
|
if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
|
|
847
|
-
|
|
1120
|
+
const index = Math.floor(acc / divisor) % modulus + bias;
|
|
1121
|
+
if ((index >= symbolCount || index < 0 && index !== bias) && bounds !== null) {
|
|
1122
|
+
refuse2(bounds, row, index, bias);
|
|
1123
|
+
}
|
|
1124
|
+
out[row] = index;
|
|
848
1125
|
}
|
|
849
1126
|
return out;
|
|
850
1127
|
}
|
|
1128
|
+
function refuse2(bounds, row, index, bias) {
|
|
1129
|
+
throw new exports.QvdCorruptedError("Symbol index out of range", {
|
|
1130
|
+
field: bounds.field,
|
|
1131
|
+
row: bounds.firstRow + row,
|
|
1132
|
+
symbolIndex: index,
|
|
1133
|
+
symbolCount: bounds.symbolCount,
|
|
1134
|
+
bias,
|
|
1135
|
+
file: bounds.file,
|
|
1136
|
+
stage: "parseIndexTable"
|
|
1137
|
+
});
|
|
1138
|
+
}
|
|
851
1139
|
function writeBitField(buffer, recordBase, geometry, value) {
|
|
852
1140
|
const { byteStart, shift, byteCount } = geometry;
|
|
853
1141
|
if (byteCount === 0) {
|
|
@@ -864,10 +1152,12 @@ function writeBitField(buffer, recordBase, geometry, value) {
|
|
|
864
1152
|
var MAX_BIT_WIDTH, POW2;
|
|
865
1153
|
var init_bitUtils = __esm({
|
|
866
1154
|
"src/util/bitUtils.js"() {
|
|
1155
|
+
init_QvdErrors();
|
|
867
1156
|
MAX_BIT_WIDTH = 31;
|
|
868
1157
|
POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
|
|
869
1158
|
__name(fieldGeometry, "fieldGeometry");
|
|
870
1159
|
__name(decodeIndexColumn, "decodeIndexColumn");
|
|
1160
|
+
__name(refuse2, "refuse");
|
|
871
1161
|
__name(writeBitField, "writeBitField");
|
|
872
1162
|
}
|
|
873
1163
|
});
|
|
@@ -1146,11 +1436,15 @@ function resetContradictedNumberFormat(numberFormat, facts) {
|
|
|
1146
1436
|
}
|
|
1147
1437
|
return numberFormat;
|
|
1148
1438
|
}
|
|
1149
|
-
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT; exports.QvdFileWriter = void 0;
|
|
1439
|
+
var OBJECT_MEMO_BASE, NUMERIC_TAGS, TEXT_TAGS, WHOLE_NUMBER_TAGS, NUMERIC_FORMATS, UNKNOWN_NUMBER_FORMAT, WRITE_CHUNK_SIZE; exports.QvdFileWriter = void 0;
|
|
1150
1440
|
var init_QvdFileWriter = __esm({
|
|
1151
1441
|
"src/QvdFileWriter.js"() {
|
|
1152
1442
|
init_QvdErrors();
|
|
1153
1443
|
init_validatePath();
|
|
1444
|
+
init_ioErrors();
|
|
1445
|
+
init_openChecked();
|
|
1446
|
+
init_replaceFile();
|
|
1447
|
+
init_optionTypes();
|
|
1154
1448
|
init_bitUtils();
|
|
1155
1449
|
init_cellRules();
|
|
1156
1450
|
init_symbolBytes();
|
|
@@ -1177,6 +1471,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1177
1471
|
UNKNOWN_NUMBER_FORMAT = Object.freeze({ Type: "UNKNOWN", nDec: "0", UseThou: "0", Fmt: "", Dec: "", Thou: "" });
|
|
1178
1472
|
__name(pruneContradictedTags, "pruneContradictedTags");
|
|
1179
1473
|
__name(resetContradictedNumberFormat, "resetContradictedNumberFormat");
|
|
1474
|
+
WRITE_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
1180
1475
|
exports.QvdFileWriter = class {
|
|
1181
1476
|
static {
|
|
1182
1477
|
__name(this, "QvdFileWriter");
|
|
@@ -1193,10 +1488,30 @@ var init_QvdFileWriter = __esm({
|
|
|
1193
1488
|
* an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
|
|
1194
1489
|
* empty value falls back to the working directory rather than removing the restriction.
|
|
1195
1490
|
* @param {Function} [options.onProgress] Optional progress callback function.
|
|
1491
|
+
* @param {boolean} [options.atomic=true] Whether to replace the destination rather than rewrite it:
|
|
1492
|
+
* the file is built beside it under a temporary name and renamed over it, so a failure leaves the
|
|
1493
|
+
* previous file exactly as it was and nothing ever reads a part-written one. False rewrites the
|
|
1494
|
+
* destination in place, which is what every version before 2.0.0 did: that keeps the file's
|
|
1495
|
+
* identity - hard links, its owner, permissions set on the file itself - needs no room for two
|
|
1496
|
+
* copies at once and no permission to create files in the directory, and destroys the previous
|
|
1497
|
+
* file the moment the write begins. A file that does not exist yet is renamed into place either
|
|
1498
|
+
* way, since there is nothing to rewrite - see `_destination`.
|
|
1499
|
+
* @param {boolean} [options.fsync=true] Whether to wait for the file's contents to reach the disk
|
|
1500
|
+
* before the write is finished. It is what carries an atomic write's promise through a power
|
|
1501
|
+
* loss - the contents are on the disk before anything is renamed, so a crash leaves the previous
|
|
1502
|
+
* file or the new one and never a damaged one - and it is the only thing that reports a failing
|
|
1503
|
+
* disk's deferred error rather than losing it. The rename itself is not flushed, so a crash just
|
|
1504
|
+
* after the call can still lose the replacement, or a file that did not exist before. False
|
|
1505
|
+
* resolves as soon as the operating system has accepted the bytes, which is faster and is what
|
|
1506
|
+
* every version before 2.0.0 did.
|
|
1196
1507
|
*/
|
|
1197
1508
|
constructor(filePath, df, options = {}) {
|
|
1198
|
-
const { allowedDir, onProgress } = options;
|
|
1199
|
-
|
|
1509
|
+
const { allowedDir, onProgress, atomic, fsync } = options;
|
|
1510
|
+
const checked = checkPath(filePath, allowedDir);
|
|
1511
|
+
this._path = checked.path;
|
|
1512
|
+
this._allowedDir = checked.base;
|
|
1513
|
+
this._atomic = booleanOption(atomic, { option: "atomic", file: this._path, whenUnset: true });
|
|
1514
|
+
this._fsync = booleanOption(fsync, { option: "fsync", file: this._path, whenUnset: true });
|
|
1200
1515
|
this._df = df;
|
|
1201
1516
|
this._onProgress = onProgress;
|
|
1202
1517
|
this._header = null;
|
|
@@ -1232,22 +1547,168 @@ var init_QvdFileWriter = __esm({
|
|
|
1232
1547
|
* Writes the data to the QVD file.
|
|
1233
1548
|
*/
|
|
1234
1549
|
async _writeData() {
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1550
|
+
assert3__default.default(this._header, "The QVD file header has not been parsed.");
|
|
1551
|
+
assert3__default.default(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
|
|
1552
|
+
assert3__default.default(this._indexBuffer, "The QVD file index table has not been parsed.");
|
|
1238
1553
|
this._emitProgress("write", 0, 1);
|
|
1239
1554
|
const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
|
|
1240
|
-
|
|
1555
|
+
const failed = rethrowAsIoError(this._path, "write");
|
|
1556
|
+
const destination = this._destination();
|
|
1557
|
+
if (destination.replace) {
|
|
1558
|
+
await this._replaceFile(destination, headerBuffer, failed);
|
|
1559
|
+
} else {
|
|
1560
|
+
await this._writeInPlace(destination, headerBuffer, failed);
|
|
1561
|
+
}
|
|
1562
|
+
this._emitProgress("write", 1, 1);
|
|
1563
|
+
}
|
|
1564
|
+
/**
|
|
1565
|
+
* What is at the destination, and therefore how it is to be written.
|
|
1566
|
+
*
|
|
1567
|
+
* Decided by one containment check, made now - immediately before anything is opened - and by
|
|
1568
|
+
* nothing else: the file that check approved is the file written, in either mode, and what the
|
|
1569
|
+
* check found there is what decides how.
|
|
1570
|
+
*
|
|
1571
|
+
* It used to stat and resolve the path again for itself, after the check. That second resolution
|
|
1572
|
+
* was #247 in the atomic write: a destination swapped for a symlink after the check was followed by
|
|
1573
|
+
* it, so the temporary file was built beside a file outside allowedDir and renamed over it. A check
|
|
1574
|
+
* reports where it went, so nothing here has to go there again.
|
|
1575
|
+
*
|
|
1576
|
+
* @return {{checked: import('./util/validatePath.js').CheckedPath, path: string,
|
|
1577
|
+
* existing: import('fs').BigIntStats|null, replace: boolean, sync: boolean}} The check, the file it
|
|
1578
|
+
* approved, what was there, whether to replace it rather than rewrite it, and whether to flush it.
|
|
1579
|
+
* @private
|
|
1580
|
+
*/
|
|
1581
|
+
_destination() {
|
|
1582
|
+
const checked = checkPath(this._path, this._allowedDir);
|
|
1583
|
+
const existing = checked.stats;
|
|
1584
|
+
const special = existing !== null && !existing.isFile();
|
|
1585
|
+
const replace = (this._atomic || existing === null) && !special;
|
|
1586
|
+
return {
|
|
1587
|
+
checked,
|
|
1588
|
+
path: checked.target,
|
|
1589
|
+
existing,
|
|
1590
|
+
replace,
|
|
1591
|
+
sync: this._fsync && !special
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
/**
|
|
1595
|
+
* Rewrites the destination where it stands - the way every version before 2.0.0 wrote.
|
|
1596
|
+
*
|
|
1597
|
+
* The file is emptied as the write begins, so from there until the last byte is written there is no
|
|
1598
|
+
* previous version left: a failure part-way through leaves a stub that no read of its rows survives,
|
|
1599
|
+
* and anything reading the path meanwhile sees however much of the new file has arrived.
|
|
1600
|
+
*
|
|
1601
|
+
* It is emptied by `openChecked` rather than by opening with `'w'`, and later than `'w'` would: once
|
|
1602
|
+
* the descriptor is known to be the file the check approved. `'w'` empties whatever the open reaches,
|
|
1603
|
+
* which is what let #247 truncate a file outside allowedDir through a symlink swapped in after the
|
|
1604
|
+
* check. The file that results is the same.
|
|
1605
|
+
*
|
|
1606
|
+
* @param {{checked: import('./util/validatePath.js').CheckedPath, sync: boolean}} destination Where
|
|
1607
|
+
* to write, from `_destination`.
|
|
1608
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1609
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1610
|
+
* @private
|
|
1611
|
+
*/
|
|
1612
|
+
async _writeInPlace(destination, headerBuffer, failed) {
|
|
1613
|
+
const fd = await openChecked(destination.checked, "rewrite", failed);
|
|
1614
|
+
await closeAfter(fd, failed, async () => {
|
|
1615
|
+
await this._writeParts(fd, headerBuffer, failed);
|
|
1616
|
+
if (destination.sync) {
|
|
1617
|
+
await fd.sync().catch(failed);
|
|
1618
|
+
}
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
* Builds the file beside the destination and renames it over it.
|
|
1623
|
+
*
|
|
1624
|
+
* Nothing touches the destination until the rename, which either replaces it or leaves it as it was,
|
|
1625
|
+
* so a write that fails at any step - a full disk, a process killed, an error from the disk itself -
|
|
1626
|
+
* costs the temporary file and nothing else. A reader of the path gets the previous file or the new
|
|
1627
|
+
* one, never a part of either, which is the other half of what the old behaviour could not promise.
|
|
1628
|
+
*
|
|
1629
|
+
* @param {{path: string, existing: import('fs').BigIntStats|null, sync: boolean}} destination Where to
|
|
1630
|
+
* write, from `_destination`.
|
|
1631
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1632
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1633
|
+
* @private
|
|
1634
|
+
*/
|
|
1635
|
+
async _replaceFile(destination, headerBuffer, failed) {
|
|
1636
|
+
const temporary = temporaryPathFor(destination.path);
|
|
1637
|
+
const mode = destination.existing === null ? void 0 : 384;
|
|
1638
|
+
const fd = await fs2__default.default.promises.open(temporary, "wx", mode).catch(failed);
|
|
1241
1639
|
try {
|
|
1242
|
-
fd
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1640
|
+
await closeAfter(fd, failed, async () => {
|
|
1641
|
+
await this._writeParts(fd, headerBuffer, failed);
|
|
1642
|
+
if (destination.existing !== null && process.platform !== "win32") {
|
|
1643
|
+
await fd.chmod(Number(destination.existing.mode) & 511).catch(failed);
|
|
1644
|
+
}
|
|
1645
|
+
if (destination.sync) {
|
|
1646
|
+
await fd.sync().catch(failed);
|
|
1647
|
+
}
|
|
1648
|
+
});
|
|
1649
|
+
await renameOver(temporary, destination.path).catch(failed);
|
|
1650
|
+
} catch (error) {
|
|
1651
|
+
const removed = await removeTemporary(temporary);
|
|
1652
|
+
const context = (
|
|
1653
|
+
/** @type {{context?: Record<string, unknown>}} */
|
|
1654
|
+
error?.context
|
|
1655
|
+
);
|
|
1656
|
+
if (!removed && context !== null && typeof context === "object" && Object.isExtensible(context)) {
|
|
1657
|
+
context.temporaryFile = temporary;
|
|
1250
1658
|
}
|
|
1659
|
+
throw error;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
/**
|
|
1663
|
+
* Writes the three parts of a QVD, in their order, into an open file.
|
|
1664
|
+
*
|
|
1665
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
1666
|
+
* @param {Buffer} headerBuffer The header and its terminator.
|
|
1667
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler.
|
|
1668
|
+
* @private
|
|
1669
|
+
*/
|
|
1670
|
+
async _writeParts(fd, headerBuffer, failed) {
|
|
1671
|
+
await this._writeRange(fd, headerBuffer, 0, failed);
|
|
1672
|
+
await this._writeRange(fd, this._symbolBuffer, headerBuffer.length, failed);
|
|
1673
|
+
await this._writeRange(fd, this._indexBuffer, headerBuffer.length + this._symbolBuffer.length, failed);
|
|
1674
|
+
}
|
|
1675
|
+
/**
|
|
1676
|
+
* Writes the whole of one buffer into the file, starting at `filePosition`.
|
|
1677
|
+
*
|
|
1678
|
+
* A write can resolve having written less than it was given, and that is all a disk that fills
|
|
1679
|
+
* part-way through one says. libuv retries a short write(2) itself, and when the retry fails it
|
|
1680
|
+
* returns the bytes that did land instead of the error - `uv__fs_write_all` in its `src/unix/fs.c`,
|
|
1681
|
+
* and `fs__write` on Windows does the same - so Node resolves with a short `bytesWritten` and never
|
|
1682
|
+
* rejects. Taking that for the whole write is how `toQvd()` resolved on a full disk and left a
|
|
1683
|
+
* truncated file: the index table, the last of the three writes, stopped part-way, and no later call
|
|
1684
|
+
* asked the disk again.
|
|
1685
|
+
*
|
|
1686
|
+
* So the rest is written until there is none, and it is the next call that reports the failure: the
|
|
1687
|
+
* disk refuses it outright, and that rejection is a QvdIOError with the system's code, ENOSPC, like
|
|
1688
|
+
* any other refused write. A write that stores nothing and reports nothing would repeat forever, so
|
|
1689
|
+
* it is a failure too, the one QvdIOError with no system code to carry.
|
|
1690
|
+
*
|
|
1691
|
+
* In bounded chunks as well, because Node refuses a single write of 2 GiB or more.
|
|
1692
|
+
*
|
|
1693
|
+
* @param {import('fs/promises').FileHandle} fd The open file.
|
|
1694
|
+
* @param {Buffer} buffer What to write.
|
|
1695
|
+
* @param {number} filePosition Where in the file it starts.
|
|
1696
|
+
* @param {(error: unknown) => never} failed The write's `rethrowAsIoError` handler, which every call
|
|
1697
|
+
* goes through.
|
|
1698
|
+
* @private
|
|
1699
|
+
*/
|
|
1700
|
+
async _writeRange(fd, buffer, filePosition, failed) {
|
|
1701
|
+
let done = 0;
|
|
1702
|
+
while (done < buffer.length) {
|
|
1703
|
+
const length = Math.min(WRITE_CHUNK_SIZE, buffer.length - done);
|
|
1704
|
+
const { bytesWritten } = await fd.write(buffer, done, length, filePosition + done).catch(failed);
|
|
1705
|
+
if (!(bytesWritten > 0)) {
|
|
1706
|
+
throw new exports.QvdIOError(
|
|
1707
|
+
`Could not write the QVD file: nothing was written at byte ${filePosition + done}, and the operating system reported no error.`,
|
|
1708
|
+
{ file: this._path, operation: "write", filePosition: filePosition + done }
|
|
1709
|
+
);
|
|
1710
|
+
}
|
|
1711
|
+
done += bytesWritten;
|
|
1251
1712
|
}
|
|
1252
1713
|
}
|
|
1253
1714
|
/**
|
|
@@ -1259,13 +1720,13 @@ var init_QvdFileWriter = __esm({
|
|
|
1259
1720
|
const existingMetadata = this._df.metadata;
|
|
1260
1721
|
const baseMetadata = existingMetadata ? {
|
|
1261
1722
|
QvBuildNo: existingMetadata.QvBuildNo || 50667,
|
|
1262
|
-
CreatorDoc: existingMetadata.CreatorDoc ||
|
|
1723
|
+
CreatorDoc: existingMetadata.CreatorDoc || crypto2__default.default.randomUUID(),
|
|
1263
1724
|
CreateUtcTime: existingMetadata.CreateUtcTime || creationDate,
|
|
1264
1725
|
SourceCreateUtcTime: existingMetadata.SourceCreateUtcTime || "",
|
|
1265
1726
|
SourceFileUtcTime: existingMetadata.SourceFileUtcTime || "",
|
|
1266
1727
|
SourceFileSize: existingMetadata.SourceFileSize || -1,
|
|
1267
1728
|
StaleUtcTime: existingMetadata.StaleUtcTime || "",
|
|
1268
|
-
TableName: existingMetadata.TableName ||
|
|
1729
|
+
TableName: existingMetadata.TableName || path2__default.default.basename(this._path, path2__default.default.extname(this._path)),
|
|
1269
1730
|
Compression: existingMetadata.Compression || "",
|
|
1270
1731
|
Comment: existingMetadata.Comment || "",
|
|
1271
1732
|
EncryptionInfo: existingMetadata.EncryptionInfo || "",
|
|
@@ -1279,13 +1740,13 @@ var init_QvdFileWriter = __esm({
|
|
|
1279
1740
|
}
|
|
1280
1741
|
} : {
|
|
1281
1742
|
QvBuildNo: 50667,
|
|
1282
|
-
CreatorDoc:
|
|
1743
|
+
CreatorDoc: crypto2__default.default.randomUUID(),
|
|
1283
1744
|
CreateUtcTime: creationDate,
|
|
1284
1745
|
SourceCreateUtcTime: "",
|
|
1285
1746
|
SourceFileUtcTime: "",
|
|
1286
1747
|
SourceFileSize: -1,
|
|
1287
1748
|
StaleUtcTime: "",
|
|
1288
|
-
TableName:
|
|
1749
|
+
TableName: path2__default.default.basename(this._path, path2__default.default.extname(this._path)),
|
|
1289
1750
|
Compression: "",
|
|
1290
1751
|
Comment: "",
|
|
1291
1752
|
EncryptionInfo: "",
|
|
@@ -1536,7 +1997,7 @@ var init_QvdFileWriter = __esm({
|
|
|
1536
1997
|
const key = keys[slot];
|
|
1537
1998
|
offset = typeof key === "number" ? writeSymbol(columnBuffer, offset, kinds[slot], key, texts[slot]) : writeSymbol(columnBuffer, offset, kinds[slot], null, key);
|
|
1538
1999
|
}
|
|
1539
|
-
|
|
2000
|
+
assert3__default.default(offset === byteLength, "A column was encoded into a different number of bytes than it was sized for.");
|
|
1540
2001
|
columnBuffers.push(columnBuffer);
|
|
1541
2002
|
this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
|
|
1542
2003
|
this._symbolCounts?.push(keys.length);
|
|
@@ -1575,9 +2036,9 @@ var init_QvdFileWriter = __esm({
|
|
|
1575
2036
|
* @private
|
|
1576
2037
|
*/
|
|
1577
2038
|
_buildIndexTable() {
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
2039
|
+
assert3__default.default(this._symbolCounts, "The QVD file symbol table has not been built.");
|
|
2040
|
+
assert3__default.default(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
|
|
2041
|
+
assert3__default.default(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
|
|
1581
2042
|
this._indexTableMetadata = [];
|
|
1582
2043
|
const columns = this._df.columns;
|
|
1583
2044
|
const data = this._df.data;
|
|
@@ -2130,6 +2591,15 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
|
|
|
2130
2591
|
stage: "parseIndexTable"
|
|
2131
2592
|
});
|
|
2132
2593
|
}
|
|
2594
|
+
if (bias < -2147483648 || bias + 2 ** bitWidth - 1 > 2 ** 31 - 1) {
|
|
2595
|
+
throw new exports.QvdCorruptedError("Bias out of range", {
|
|
2596
|
+
field: field["FieldName"],
|
|
2597
|
+
bias,
|
|
2598
|
+
bitWidth,
|
|
2599
|
+
file: filePath,
|
|
2600
|
+
stage: "parseIndexTable"
|
|
2601
|
+
});
|
|
2602
|
+
}
|
|
2133
2603
|
const recordSizeInBits = recordSize * 8;
|
|
2134
2604
|
if (bitOffset + bitWidth > recordSizeInBits) {
|
|
2135
2605
|
throw new exports.QvdCorruptedError("Bit field extends beyond record size", {
|
|
@@ -2257,7 +2727,10 @@ function parseFieldSymbols(symbolBuffer, start, end, keep, fieldName, filePath)
|
|
|
2257
2727
|
}
|
|
2258
2728
|
return { numbers, texts };
|
|
2259
2729
|
}
|
|
2260
|
-
|
|
2730
|
+
function countFieldSymbols(symbolBuffer, start, end, fieldName, filePath) {
|
|
2731
|
+
return parseFieldSymbols(symbolBuffer, start, end, DECODE_NOTHING, fieldName, filePath).numbers.length;
|
|
2732
|
+
}
|
|
2733
|
+
var MAX_TEXT_BYTES, DECODE_NOTHING;
|
|
2261
2734
|
var init_symbolParser = __esm({
|
|
2262
2735
|
"src/util/symbolParser.js"() {
|
|
2263
2736
|
init_QvdErrors();
|
|
@@ -2265,6 +2738,8 @@ var init_symbolParser = __esm({
|
|
|
2265
2738
|
__name(textEnd, "textEnd");
|
|
2266
2739
|
__name(overflow, "overflow");
|
|
2267
2740
|
__name(parseFieldSymbols, "parseFieldSymbols");
|
|
2741
|
+
DECODE_NOTHING = /* @__PURE__ */ new Set();
|
|
2742
|
+
__name(countFieldSymbols, "countFieldSymbols");
|
|
2268
2743
|
}
|
|
2269
2744
|
});
|
|
2270
2745
|
|
|
@@ -2696,9 +3171,9 @@ var init_QvdColumnTable = __esm({
|
|
|
2696
3171
|
* read switches to two-pass filtering.
|
|
2697
3172
|
* @return {Promise<QvdColumnTable>} The file, as columns.
|
|
2698
3173
|
*/
|
|
2699
|
-
static async fromQvd(
|
|
3174
|
+
static async fromQvd(path5, options = {}) {
|
|
2700
3175
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
2701
|
-
const reader = new QvdFileReader2(
|
|
3176
|
+
const reader = new QvdFileReader2(path5, {
|
|
2702
3177
|
...readerOptionsFrom(options),
|
|
2703
3178
|
// This read builds no rows, so the memory guard must not charge it for them. A columnar
|
|
2704
3179
|
// read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
|
|
@@ -2766,22 +3241,26 @@ var QvdFileReader_exports = {};
|
|
|
2766
3241
|
__export(QvdFileReader_exports, {
|
|
2767
3242
|
QvdFileReader: () => exports.QvdFileReader
|
|
2768
3243
|
});
|
|
2769
|
-
function
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
3244
|
+
async function* chunksFrom(handle, chunkSize, failed) {
|
|
3245
|
+
let position = 0;
|
|
3246
|
+
for (; ; ) {
|
|
3247
|
+
const buffer = Buffer.alloc(chunkSize);
|
|
3248
|
+
const { bytesRead } = await handle.read(buffer, 0, chunkSize, position).catch(failed);
|
|
3249
|
+
if (bytesRead === 0) {
|
|
3250
|
+
return;
|
|
3251
|
+
}
|
|
3252
|
+
yield bytesRead === chunkSize ? buffer : Buffer.from(buffer.subarray(0, bytesRead));
|
|
3253
|
+
position += bytesRead;
|
|
3254
|
+
}
|
|
2778
3255
|
}
|
|
2779
|
-
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS; exports.QvdFileReader = void 0;
|
|
3256
|
+
var MAX_HEADER_SIZE, READ_CHUNK_SIZE, ANALYSIS_SLICE_ROWS, COUNT_SYMBOLS_PAST; exports.QvdFileReader = void 0;
|
|
2780
3257
|
var init_QvdFileReader = __esm({
|
|
2781
3258
|
"src/QvdFileReader.js"() {
|
|
2782
3259
|
init_QvdDataFrame();
|
|
2783
3260
|
init_QvdErrors();
|
|
2784
3261
|
init_validatePath();
|
|
3262
|
+
init_openChecked();
|
|
3263
|
+
init_ioErrors();
|
|
2785
3264
|
init_bitUtils();
|
|
2786
3265
|
init_memoryUtils();
|
|
2787
3266
|
init_validationUtils();
|
|
@@ -2792,7 +3271,8 @@ var init_QvdFileReader = __esm({
|
|
|
2792
3271
|
MAX_HEADER_SIZE = 16 * 1024 * 1024;
|
|
2793
3272
|
READ_CHUNK_SIZE = 512 * 1024 * 1024;
|
|
2794
3273
|
ANALYSIS_SLICE_ROWS = 65536;
|
|
2795
|
-
|
|
3274
|
+
COUNT_SYMBOLS_PAST = 65536;
|
|
3275
|
+
__name(chunksFrom, "chunksFrom");
|
|
2796
3276
|
exports.QvdFileReader = class {
|
|
2797
3277
|
static {
|
|
2798
3278
|
__name(this, "QvdFileReader");
|
|
@@ -2853,7 +3333,9 @@ var init_QvdFileReader = __esm({
|
|
|
2853
3333
|
signal
|
|
2854
3334
|
} = options;
|
|
2855
3335
|
this._materialisesRows = materialisesRows;
|
|
2856
|
-
|
|
3336
|
+
const checked = checkPath(filePath, allowedDir);
|
|
3337
|
+
this._path = checked.path;
|
|
3338
|
+
this._allowedDir = checked.base;
|
|
2857
3339
|
this._duals = normaliseDuals(duals, this._path);
|
|
2858
3340
|
this._coerceNumericStrings = normaliseCoerceNumericStrings(coerceNumericStrings, this._path);
|
|
2859
3341
|
this._memorySafetyFactor = memorySafetyFactor;
|
|
@@ -2938,18 +3420,21 @@ var init_QvdFileReader = __esm({
|
|
|
2938
3420
|
* two-pass path exist for.
|
|
2939
3421
|
*
|
|
2940
3422
|
* Algorithm for a windowed read:
|
|
2941
|
-
* 1.
|
|
3423
|
+
* 1. Read the file a chunk at a time until the XML header delimiter is found
|
|
2942
3424
|
* 2. Parse header to determine symbol table and index table locations
|
|
2943
3425
|
* 3. Calculate bytes needed: header + full symbol table + partial index table
|
|
2944
|
-
* 4. Read only those calculated bytes
|
|
3426
|
+
* 4. Read only those calculated bytes, by position
|
|
2945
3427
|
* 5. Rest of parsing proceeds normally with limited data
|
|
2946
3428
|
*
|
|
2947
3429
|
* WHY THIS APPROACH:
|
|
2948
3430
|
* - Symbol table must be fully loaded (contains all unique values)
|
|
2949
3431
|
* - Index table can be partially loaded (only rows we need)
|
|
2950
|
-
* -
|
|
3432
|
+
* - Reading chunks to find the header is efficient for unknown header sizes
|
|
2951
3433
|
* - Direct byte-range reading for remaining data is fastest
|
|
2952
3434
|
*
|
|
3435
|
+
* All of it goes through one handle, opened once, on the file the containment check approved. See
|
|
3436
|
+
* `chunksFrom` and `openChecked` for why a read no longer opens the path more than once.
|
|
3437
|
+
*
|
|
2953
3438
|
* A window with a non-zero `offset` reads two ranges rather than one: the header and symbol
|
|
2954
3439
|
* table from the front of the file, and the window's records from wherever they sit. The bytes
|
|
2955
3440
|
* between are never read, which is what makes `{offset: 1_700_000, limit: 100}` on the taxi
|
|
@@ -2967,49 +3452,49 @@ var init_QvdFileReader = __esm({
|
|
|
2967
3452
|
async _readData(window = { offset: 0, limit: null }, headerOnly = false, liveRows = null) {
|
|
2968
3453
|
this._throwIfAborted();
|
|
2969
3454
|
this._emitProgress("read", 0, 1);
|
|
3455
|
+
const failed = rethrowAsIoError(this._path, "read");
|
|
3456
|
+
const handle = await openChecked(checkPath(this._path, this._allowedDir), "read", failed);
|
|
3457
|
+
await closeAfter(handle, failed, () => this._readFrom(handle, window, headerOnly, liveRows, failed));
|
|
3458
|
+
}
|
|
3459
|
+
/**
|
|
3460
|
+
* Reads what `_readData` was asked for, through the handle it opened.
|
|
3461
|
+
*
|
|
3462
|
+
* @param {import('fs/promises').FileHandle} handle The open file.
|
|
3463
|
+
* @param {QvdRowWindow} window The rows to read.
|
|
3464
|
+
* @param {boolean} headerOnly Stop once the XML header has been read.
|
|
3465
|
+
* @param {{rows: number, perChunk: number}|null} liveRows Rows held at one instant - see `_prepare`.
|
|
3466
|
+
* @param {(error: unknown) => never} failed The read's `rethrowAsIoError` handler.
|
|
3467
|
+
* @private
|
|
3468
|
+
*/
|
|
3469
|
+
async _readFrom(handle, window, headerOnly, liveRows, failed) {
|
|
2970
3470
|
const HEADER_DELIMITER = "\r\n\0";
|
|
2971
3471
|
const CHUNK_SIZE = 64 * 1024;
|
|
2972
|
-
const stream = fs__default.default.createReadStream(this._path, {
|
|
2973
|
-
highWaterMark: CHUNK_SIZE
|
|
2974
|
-
});
|
|
2975
3472
|
const headerChunks = [];
|
|
2976
3473
|
let headerBytes = 0;
|
|
2977
3474
|
let tail = Buffer.alloc(0);
|
|
2978
3475
|
let headerDelimiterIndex = -1;
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
stream.destroy();
|
|
2989
|
-
break;
|
|
2990
|
-
}
|
|
2991
|
-
tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
|
|
2992
|
-
if (headerBytes > MAX_HEADER_SIZE) {
|
|
2993
|
-
stream.destroy();
|
|
2994
|
-
throw new exports.QvdCorruptedError(
|
|
2995
|
-
`The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
|
|
2996
|
-
{
|
|
2997
|
-
file: this._path,
|
|
2998
|
-
bytesSearched: headerBytes,
|
|
2999
|
-
maxHeaderSize: MAX_HEADER_SIZE,
|
|
3000
|
-
stage: "readData"
|
|
3001
|
-
}
|
|
3002
|
-
);
|
|
3003
|
-
}
|
|
3476
|
+
for await (const chunk of chunksFrom(handle, CHUNK_SIZE, failed)) {
|
|
3477
|
+
const chunkStart = headerBytes;
|
|
3478
|
+
const searchBuffer = tail.length > 0 ? Buffer.concat([tail, chunk]) : chunk;
|
|
3479
|
+
const foundInSearch = searchBuffer.indexOf(HEADER_DELIMITER);
|
|
3480
|
+
headerChunks.push(chunk);
|
|
3481
|
+
headerBytes += chunk.length;
|
|
3482
|
+
if (foundInSearch !== -1) {
|
|
3483
|
+
headerDelimiterIndex = chunkStart - tail.length + foundInSearch;
|
|
3484
|
+
break;
|
|
3004
3485
|
}
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3486
|
+
tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
|
|
3487
|
+
if (headerBytes > MAX_HEADER_SIZE) {
|
|
3488
|
+
throw new exports.QvdCorruptedError(
|
|
3489
|
+
`The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
|
|
3490
|
+
{
|
|
3491
|
+
file: this._path,
|
|
3492
|
+
bytesSearched: headerBytes,
|
|
3493
|
+
maxHeaderSize: MAX_HEADER_SIZE,
|
|
3494
|
+
stage: "readData"
|
|
3495
|
+
}
|
|
3496
|
+
);
|
|
3010
3497
|
}
|
|
3011
|
-
} finally {
|
|
3012
|
-
await closeReadStream(stream);
|
|
3013
3498
|
}
|
|
3014
3499
|
if (headerDelimiterIndex === -1) {
|
|
3015
3500
|
throw new exports.QvdCorruptedError(
|
|
@@ -3050,9 +3535,9 @@ var init_QvdFileReader = __esm({
|
|
|
3050
3535
|
(value) => Number.isSafeInteger(value) && value >= 0
|
|
3051
3536
|
);
|
|
3052
3537
|
if (headerNumbersUsable) {
|
|
3053
|
-
const { size:
|
|
3054
|
-
this._fileSize =
|
|
3055
|
-
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <=
|
|
3538
|
+
const { size: fileSize2 } = await handle.stat().catch(failed);
|
|
3539
|
+
this._fileSize = fileSize2;
|
|
3540
|
+
this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize2;
|
|
3056
3541
|
}
|
|
3057
3542
|
const resolved = headerNumbersUsable ? resolveWindow(window, totalRows) : { offset: 0, limit: 0 };
|
|
3058
3543
|
const windowRows = resolved.limit;
|
|
@@ -3069,7 +3554,7 @@ var init_QvdFileReader = __esm({
|
|
|
3069
3554
|
);
|
|
3070
3555
|
}
|
|
3071
3556
|
if (window.offset === 0 && window.limit === null) {
|
|
3072
|
-
this._buffer = await
|
|
3557
|
+
this._buffer = await handle.readFile().catch(failed);
|
|
3073
3558
|
this._fileSize = this._buffer.length;
|
|
3074
3559
|
this._bufferFirstRow = 0;
|
|
3075
3560
|
this._emitProgress("read", 1, 1);
|
|
@@ -3095,34 +3580,29 @@ var init_QvdFileReader = __esm({
|
|
|
3095
3580
|
const indexTableBytesToRead = rowsToLoad * recordSize;
|
|
3096
3581
|
const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
|
|
3097
3582
|
const fileBytesRequired = indexTableOffset + skippedIndexBytes + indexTableBytesToRead;
|
|
3098
|
-
const
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
stage: "readData"
|
|
3108
|
-
});
|
|
3109
|
-
}
|
|
3110
|
-
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
3111
|
-
await this._readRange(fd, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3112
|
-
if (indexTableBytesToRead > 0) {
|
|
3113
|
-
await this._readRange(
|
|
3114
|
-
fd,
|
|
3115
|
-
indexTableOffset,
|
|
3116
|
-
indexTableBytesToRead,
|
|
3117
|
-
indexTableOffset + skippedIndexBytes,
|
|
3118
|
-
fileSize,
|
|
3119
|
-
fileBytesRequired
|
|
3120
|
-
);
|
|
3121
|
-
}
|
|
3122
|
-
this._bufferFirstRow = resolved.offset;
|
|
3123
|
-
} finally {
|
|
3124
|
-
await fd.close();
|
|
3583
|
+
const { size: fileSize } = await handle.stat().catch(failed);
|
|
3584
|
+
this._fileSize = fileSize;
|
|
3585
|
+
if (fileBytesRequired > fileSize) {
|
|
3586
|
+
throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
|
|
3587
|
+
file: this._path,
|
|
3588
|
+
fileSize,
|
|
3589
|
+
requiredBytes: fileBytesRequired,
|
|
3590
|
+
stage: "readData"
|
|
3591
|
+
});
|
|
3125
3592
|
}
|
|
3593
|
+
this._buffer = Buffer.alloc(totalBytesToRead);
|
|
3594
|
+
await this._readRange(handle, 0, indexTableOffset, 0, fileSize, totalBytesToRead);
|
|
3595
|
+
if (indexTableBytesToRead > 0) {
|
|
3596
|
+
await this._readRange(
|
|
3597
|
+
handle,
|
|
3598
|
+
indexTableOffset,
|
|
3599
|
+
indexTableBytesToRead,
|
|
3600
|
+
indexTableOffset + skippedIndexBytes,
|
|
3601
|
+
fileSize,
|
|
3602
|
+
fileBytesRequired
|
|
3603
|
+
);
|
|
3604
|
+
}
|
|
3605
|
+
this._bufferFirstRow = resolved.offset;
|
|
3126
3606
|
this._emitProgress("read", 1, 1);
|
|
3127
3607
|
}
|
|
3128
3608
|
/**
|
|
@@ -3141,11 +3621,12 @@ var init_QvdFileReader = __esm({
|
|
|
3141
3621
|
* @private
|
|
3142
3622
|
*/
|
|
3143
3623
|
async _readRange(fd, bufferOffset, byteCount, filePosition, fileSize, requiredBytes) {
|
|
3144
|
-
|
|
3624
|
+
assert3__default.default(this._buffer, "The read buffer has not been allocated.");
|
|
3625
|
+
const failed = rethrowAsIoError(this._path, "read");
|
|
3145
3626
|
let done = 0;
|
|
3146
3627
|
while (done < byteCount) {
|
|
3147
3628
|
const length = Math.min(READ_CHUNK_SIZE, byteCount - done);
|
|
3148
|
-
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done);
|
|
3629
|
+
const { bytesRead } = await fd.read(this._buffer, bufferOffset + done, length, filePosition + done).catch(failed);
|
|
3149
3630
|
if (bytesRead === 0) {
|
|
3150
3631
|
throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
|
|
3151
3632
|
file: this._path,
|
|
@@ -3192,6 +3673,7 @@ var init_QvdFileReader = __esm({
|
|
|
3192
3673
|
}
|
|
3193
3674
|
const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
|
|
3194
3675
|
const headerBuffer = this._buffer.subarray(headerBeginIndex, headerEndIndex);
|
|
3676
|
+
this._fieldBitMetadataValidated = false;
|
|
3195
3677
|
this._header = await xml2__default.default.parseStringPromise(headerBuffer.toString(), { explicitArray: false });
|
|
3196
3678
|
if (!this._header) {
|
|
3197
3679
|
throw new exports.QvdParseError("The XML header could not be parsed.", {
|
|
@@ -3237,8 +3719,8 @@ var init_QvdFileReader = __esm({
|
|
|
3237
3719
|
* @param {QvdRowWindow} window The rows of interest, as file row indices.
|
|
3238
3720
|
* @param {string} stage Stage name for any error raised here.
|
|
3239
3721
|
* @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
|
|
3240
|
-
* indexBuffer: Buffer}} The record geometry. `indexBuffer` starts at the
|
|
3241
|
-
* record, so the decoder always counts from zero.
|
|
3722
|
+
* indexBuffer: Buffer, firstRow: number}} The record geometry. `indexBuffer` starts at the
|
|
3723
|
+
* window's first record, file row `firstRow`, so the decoder always counts from zero.
|
|
3242
3724
|
* @private
|
|
3243
3725
|
*/
|
|
3244
3726
|
_planIndexTable(window, stage) {
|
|
@@ -3280,11 +3762,11 @@ var init_QvdFileReader = __esm({
|
|
|
3280
3762
|
}
|
|
3281
3763
|
this._fieldBitMetadataValidated = true;
|
|
3282
3764
|
}
|
|
3283
|
-
|
|
3765
|
+
assert3__default.default(
|
|
3284
3766
|
rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
|
|
3285
3767
|
`The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
|
|
3286
3768
|
);
|
|
3287
|
-
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
|
|
3769
|
+
return { fields, recordSize, totalRows, rowsToLoad, indexBuffer, firstRow };
|
|
3288
3770
|
}
|
|
3289
3771
|
/**
|
|
3290
3772
|
* Analyzes the index table to determine which symbols are actually needed.
|
|
@@ -3311,6 +3793,9 @@ var init_QvdFileReader = __esm({
|
|
|
3311
3793
|
const bitOffset = parseInt(field["BitOffset"], 10);
|
|
3312
3794
|
const bitWidth = parseInt(field["BitWidth"], 10);
|
|
3313
3795
|
const bias = parseInt(field["Bias"], 10);
|
|
3796
|
+
const length = parseInt(field["Length"], 10);
|
|
3797
|
+
let indexLimit = Number.isSafeInteger(length) && length >= 0 ? Math.ceil(length / 2) : Infinity;
|
|
3798
|
+
let counted = false;
|
|
3314
3799
|
for (let first = 0; first < rowsToLoad; first += sliceRows) {
|
|
3315
3800
|
const count = Math.min(sliceRows, rowsToLoad - first);
|
|
3316
3801
|
decodeIndexColumn(
|
|
@@ -3323,15 +3808,53 @@ var init_QvdFileReader = __esm({
|
|
|
3323
3808
|
column
|
|
3324
3809
|
);
|
|
3325
3810
|
for (let row = 0; row < count; row++) {
|
|
3326
|
-
if (column[row] >= 0) {
|
|
3811
|
+
if (column[row] >= 0 && column[row] < indexLimit) {
|
|
3327
3812
|
needed.add(column[row]);
|
|
3328
3813
|
}
|
|
3329
3814
|
}
|
|
3815
|
+
if (!counted && needed.size > COUNT_SYMBOLS_PAST) {
|
|
3816
|
+
counted = true;
|
|
3817
|
+
indexLimit = Math.min(indexLimit, this._countFieldSymbols(field));
|
|
3818
|
+
for (const index of needed) {
|
|
3819
|
+
if (index >= indexLimit) {
|
|
3820
|
+
needed.delete(index);
|
|
3821
|
+
}
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3330
3824
|
}
|
|
3331
3825
|
this._emitProgress("symbol-analysis", position + 1, fields.length);
|
|
3332
3826
|
});
|
|
3333
3827
|
return symbolUsage;
|
|
3334
3828
|
}
|
|
3829
|
+
/**
|
|
3830
|
+
* How many symbols a field holds, for the symbol-usage pass, which runs before the symbols are parsed.
|
|
3831
|
+
*
|
|
3832
|
+
* The count is `countFieldSymbols`, the parse itself told to decode nothing, so it is the count
|
|
3833
|
+
* `_parseSymbolTable` will produce and `_parseIndexTable` will check against. The field's area is
|
|
3834
|
+
* validated first, as `_parseSymbolTable` would, so a damaged `Offset` or `Length` is reported the
|
|
3835
|
+
* same way wherever it is met.
|
|
3836
|
+
*
|
|
3837
|
+
* @param {any} field The field's header.
|
|
3838
|
+
* @return {number} Its symbols.
|
|
3839
|
+
* @throws {QvdCorruptedError} If the area is not inside the symbol table, or a symbol runs past it.
|
|
3840
|
+
* @private
|
|
3841
|
+
*/
|
|
3842
|
+
_countFieldSymbols(field) {
|
|
3843
|
+
assert3__default.default(
|
|
3844
|
+
this._buffer && this._symbolTableOffset && this._indexTableOffset,
|
|
3845
|
+
"The QVD file has not been read before its symbols were counted."
|
|
3846
|
+
);
|
|
3847
|
+
const symbolBuffer = this._buffer.subarray(this._symbolTableOffset, this._indexTableOffset);
|
|
3848
|
+
validateFieldMetadata(field, symbolBuffer.length, this._path);
|
|
3849
|
+
const offset = parseInt(field["Offset"], 10);
|
|
3850
|
+
return countFieldSymbols(
|
|
3851
|
+
symbolBuffer,
|
|
3852
|
+
offset,
|
|
3853
|
+
offset + parseInt(field["Length"], 10),
|
|
3854
|
+
field["FieldName"],
|
|
3855
|
+
this._path
|
|
3856
|
+
);
|
|
3857
|
+
}
|
|
3335
3858
|
/**
|
|
3336
3859
|
* Parses the symbol table of the QVD file. This method is part of the parsing process
|
|
3337
3860
|
* and should not be called directly.
|
|
@@ -3417,12 +3940,19 @@ var init_QvdFileReader = __esm({
|
|
|
3417
3940
|
* `base += recordSize`, so decoding rows k to k+n is a question of where the buffer slice starts
|
|
3418
3941
|
* and how many iterations run. Nothing about the decoder changed to support it.
|
|
3419
3942
|
*
|
|
3943
|
+
* Every index is checked as it is decoded. One that addresses neither a symbol of its field nor
|
|
3944
|
+
* NULL throws a `QvdCorruptedError` naming the file row (#125). Here and not where rows or columns
|
|
3945
|
+
* are built, because every read decodes through this method and a columnar read hands its codes
|
|
3946
|
+
* straight to the caller. Rows outside the window are not decoded, so they are not checked.
|
|
3947
|
+
*
|
|
3420
3948
|
* @param {QvdRowWindow} window The rows to decode.
|
|
3949
|
+
* @throws {QvdCorruptedError} If an index in the window addresses neither a symbol nor NULL.
|
|
3421
3950
|
*/
|
|
3422
3951
|
async _parseIndexTable(window) {
|
|
3423
|
-
const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(window, "parseIndexTable");
|
|
3424
|
-
this.
|
|
3425
|
-
|
|
3952
|
+
const { fields, recordSize, rowsToLoad, indexBuffer, firstRow } = this._planIndexTable(window, "parseIndexTable");
|
|
3953
|
+
assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
3954
|
+
const symbolTable = this._symbolTable;
|
|
3955
|
+
const columns = fields.map((field, position) => {
|
|
3426
3956
|
this._throwIfAborted();
|
|
3427
3957
|
const column = decodeIndexColumn(
|
|
3428
3958
|
indexBuffer,
|
|
@@ -3431,11 +3961,14 @@ var init_QvdFileReader = __esm({
|
|
|
3431
3961
|
parseInt(field["BitOffset"], 10),
|
|
3432
3962
|
parseInt(field["BitWidth"], 10),
|
|
3433
3963
|
parseInt(field["Bias"], 10),
|
|
3434
|
-
new Int32Array(rowsToLoad)
|
|
3964
|
+
new Int32Array(rowsToLoad),
|
|
3965
|
+
{ symbolCount: symbolTable[position].numbers.length, field: field["FieldName"], file: this._path, firstRow }
|
|
3435
3966
|
);
|
|
3436
3967
|
this._emitProgress("index-table", position + 1, fields.length);
|
|
3437
3968
|
return column;
|
|
3438
3969
|
});
|
|
3970
|
+
this._indexColumns = columns;
|
|
3971
|
+
this._rowsDecoded = rowsToLoad;
|
|
3439
3972
|
}
|
|
3440
3973
|
/**
|
|
3441
3974
|
* Reads the file's schema and header metadata, without touching the symbol or index tables.
|
|
@@ -3457,7 +3990,7 @@ var init_QvdFileReader = __esm({
|
|
|
3457
3990
|
await this._parseHeader();
|
|
3458
3991
|
this._emitProgress("header", 1, 1);
|
|
3459
3992
|
this._throwIfAborted();
|
|
3460
|
-
|
|
3993
|
+
assert3__default.default(this._header, "The QVD file header has not been parsed.");
|
|
3461
3994
|
const header = this._header["QvdTableHeader"];
|
|
3462
3995
|
let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
|
|
3463
3996
|
if (!Array.isArray(fields)) {
|
|
@@ -3521,9 +4054,10 @@ var init_QvdFileReader = __esm({
|
|
|
3521
4054
|
* Shares every step with `load()` up to the point where rows would be built - see `_prepare`.
|
|
3522
4055
|
* What it keeps instead is what the decoder already produced: one `Int32Array` of stored
|
|
3523
4056
|
* indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
|
|
3524
|
-
* fixture that is
|
|
4057
|
+
* fixture that is 133 MiB against the 352 MiB `data` retains, because a column costs four
|
|
3525
4058
|
* bytes per row rather than a boxed value per cell, and the symbols are a few thousand
|
|
3526
|
-
* entries shared across every row that uses them.
|
|
4059
|
+
* entries shared across every row that uses them. The codes' storage lives outside the V8
|
|
4060
|
+
* heap: 3 MiB of the 133 is on it.
|
|
3527
4061
|
*
|
|
3528
4062
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} [window]
|
|
3529
4063
|
* The rows to decode, in the same spellings `load()` accepts.
|
|
@@ -3534,7 +4068,7 @@ var init_QvdFileReader = __esm({
|
|
|
3534
4068
|
const prepared = await this._prepare(rows, null, true);
|
|
3535
4069
|
await this._parseIndexTable({ offset: prepared.offset, limit: prepared.rowsAvailable });
|
|
3536
4070
|
const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
|
|
3537
|
-
|
|
4071
|
+
assert3__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
3538
4072
|
return new QvdColumnTable2({
|
|
3539
4073
|
columns: prepared.columns,
|
|
3540
4074
|
codesByField: this._indexColumns,
|
|
@@ -3563,7 +4097,8 @@ var init_QvdFileReader = __esm({
|
|
|
3563
4097
|
* `chunkSize * 2`, and why the heap it needs is twice what one chunk suggests.
|
|
3564
4098
|
*
|
|
3565
4099
|
* A window covering no rows yields nothing at all, rather than one empty frame - so
|
|
3566
|
-
* `for await` over an exhausted offset does nothing, which is what a paging loop wants.
|
|
4100
|
+
* `for await` over an exhausted offset does nothing, which is what a paging loop wants. Its header
|
|
4101
|
+
* is still checked, as every read of a file's rows checks it.
|
|
3567
4102
|
*
|
|
3568
4103
|
* @param {number|null|{offset?: number, limit?: number|null, maxRows?: number|null}} window
|
|
3569
4104
|
* The rows to cover, in the same spellings `load()` accepts.
|
|
@@ -3581,6 +4116,10 @@ var init_QvdFileReader = __esm({
|
|
|
3581
4116
|
const liveRows = { rows: chunkSize * 2, perChunk: 2 };
|
|
3582
4117
|
const rows = normaliseWindow(window, this._path);
|
|
3583
4118
|
const prepared = await this._prepare(rows, liveRows);
|
|
4119
|
+
if (prepared.rowsAvailable === 0) {
|
|
4120
|
+
this._planIndexTable({ offset: prepared.offset, limit: 0 }, "parseIndexTable");
|
|
4121
|
+
return;
|
|
4122
|
+
}
|
|
3584
4123
|
for (let done = 0; done < prepared.rowsAvailable; done += chunkSize) {
|
|
3585
4124
|
this._throwIfAborted();
|
|
3586
4125
|
const count = Math.min(chunkSize, prepared.rowsAvailable - done);
|
|
@@ -3631,7 +4170,7 @@ var init_QvdFileReader = __esm({
|
|
|
3631
4170
|
await this._parseHeader();
|
|
3632
4171
|
this._emitProgress("header", 1, 1);
|
|
3633
4172
|
this._throwIfAborted();
|
|
3634
|
-
|
|
4173
|
+
assert3__default.default(this._header, "The QVD file header has not been parsed.");
|
|
3635
4174
|
const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
|
|
3636
4175
|
const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
|
|
3637
4176
|
const resolved = resolveWindow(window, totalRows);
|
|
@@ -3645,9 +4184,9 @@ var init_QvdFileReader = __esm({
|
|
|
3645
4184
|
}
|
|
3646
4185
|
}
|
|
3647
4186
|
await this._parseSymbolTable(symbolsToKeep, rowsAvailable, liveRows);
|
|
3648
|
-
|
|
4187
|
+
assert3__default.default(this._symbolTable, "The QVD file symbol table has not been parsed.");
|
|
3649
4188
|
this._throwIfAborted();
|
|
3650
|
-
|
|
4189
|
+
assert3__default.default(this._selectedFields, "The QVD file fields have not been resolved.");
|
|
3651
4190
|
const resolvedByField = [];
|
|
3652
4191
|
const halvesByField = [];
|
|
3653
4192
|
const entries = [];
|
|
@@ -3708,7 +4247,7 @@ var init_QvdFileReader = __esm({
|
|
|
3708
4247
|
* @private
|
|
3709
4248
|
*/
|
|
3710
4249
|
_buildRows(resolvedByField, progressBase, progressTotal) {
|
|
3711
|
-
|
|
4250
|
+
assert3__default.default(this._indexColumns, "The QVD file index table has not been parsed.");
|
|
3712
4251
|
const indexColumns = this._indexColumns;
|
|
3713
4252
|
const fieldCount = indexColumns.length;
|
|
3714
4253
|
const rowCount = this._rowsDecoded;
|
|
@@ -4313,14 +4852,31 @@ var init_QvdDataFrame = __esm({
|
|
|
4313
4852
|
* volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
|
|
4314
4853
|
* back to the working directory rather than removing the restriction.
|
|
4315
4854
|
* @param {Function} [options.onProgress] Optional progress callback function that receives progress updates during write operations.
|
|
4855
|
+
* @param {boolean} [options.atomic=true] Whether to replace the file rather than rewrite it. The QVD
|
|
4856
|
+
* is built beside it under a temporary name and renamed over it, so a write that fails leaves the
|
|
4857
|
+
* previous file exactly as it was, and a reader of the path - a Qlik reload, say - sees the old
|
|
4858
|
+
* file or the new one and never a part of either. False rewrites the file where it stands, as
|
|
4859
|
+
* every version before 2.0.0 did: that keeps its identity, including hard links, its owner and
|
|
4860
|
+
* permissions set on the file itself, and needs neither room for two copies nor permission to
|
|
4861
|
+
* create files in the directory. It also destroys the previous file as the write begins. A file
|
|
4862
|
+
* that does not exist yet is renamed into place either way, since there is nothing to rewrite.
|
|
4863
|
+
* @param {boolean} [options.fsync=true] Whether to wait for the contents to reach the disk before
|
|
4864
|
+
* resolving. It is what carries an atomic write's promise through a power loss - the contents are
|
|
4865
|
+
* on the disk before anything is renamed, so a crash leaves the previous file or the new one and
|
|
4866
|
+
* never a damaged one - and what reports a failing disk's deferred error instead of losing it. The
|
|
4867
|
+
* rename itself is not flushed, so a crash just after the call can still lose the replacement, or
|
|
4868
|
+
* a file that did not exist before. False resolves once the operating system has accepted the
|
|
4869
|
+
* bytes, which is faster and is what every version before 2.0.0 did.
|
|
4316
4870
|
*/
|
|
4317
|
-
async toQvd(
|
|
4871
|
+
async toQvd(path5, options = {}) {
|
|
4318
4872
|
const { QvdFileWriter: QvdFileWriter2 } = await Promise.resolve().then(() => (init_QvdFileWriter(), QvdFileWriter_exports));
|
|
4319
4873
|
const writerOptions = {
|
|
4320
4874
|
allowedDir: options.allowedDir,
|
|
4321
|
-
onProgress: options.onProgress
|
|
4875
|
+
onProgress: options.onProgress,
|
|
4876
|
+
atomic: options.atomic,
|
|
4877
|
+
fsync: options.fsync
|
|
4322
4878
|
};
|
|
4323
|
-
await new QvdFileWriter2(
|
|
4879
|
+
await new QvdFileWriter2(path5, this, writerOptions).save();
|
|
4324
4880
|
}
|
|
4325
4881
|
/**
|
|
4326
4882
|
* Loads a QVD file and returns its data frame.
|
|
@@ -4373,9 +4929,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4373
4929
|
* is not one of its modes, or if `coerceNumericStrings` is not a boolean.
|
|
4374
4930
|
* @return {Promise<QvdDataFrame>} The data frame of the QVD file.
|
|
4375
4931
|
*/
|
|
4376
|
-
static async fromQvd(
|
|
4932
|
+
static async fromQvd(path5, options = {}) {
|
|
4377
4933
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4378
|
-
return await new QvdFileReader2(
|
|
4934
|
+
return await new QvdFileReader2(path5, readerOptionsFrom(options)).load(windowFrom(options));
|
|
4379
4935
|
}
|
|
4380
4936
|
/**
|
|
4381
4937
|
* Reads a QVD file in chunks, as an async generator of data frames.
|
|
@@ -4422,9 +4978,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4422
4978
|
* windowed read switches to two-pass filtering.
|
|
4423
4979
|
* @return {AsyncGenerator<QvdDataFrame>} The chunks, in file order.
|
|
4424
4980
|
*/
|
|
4425
|
-
static async *iterate(
|
|
4981
|
+
static async *iterate(path5, options = {}) {
|
|
4426
4982
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4427
|
-
const reader = new QvdFileReader2(
|
|
4983
|
+
const reader = new QvdFileReader2(path5, readerOptionsFrom(options));
|
|
4428
4984
|
yield* reader.iterateRows(windowFrom(options), options.chunkSize === void 0 ? 1e5 : options.chunkSize);
|
|
4429
4985
|
}
|
|
4430
4986
|
/**
|
|
@@ -4455,9 +5011,9 @@ var init_QvdDataFrame = __esm({
|
|
|
4455
5011
|
* @param {AbortSignal} [options.signal] Cancels the read, rejecting with `signal.reason`.
|
|
4456
5012
|
* @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
|
|
4457
5013
|
*/
|
|
4458
|
-
static async readMetadata(
|
|
5014
|
+
static async readMetadata(path5, options = {}) {
|
|
4459
5015
|
const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
|
|
4460
|
-
return await new QvdFileReader2(
|
|
5016
|
+
return await new QvdFileReader2(path5, metadataOptionsFrom(options)).loadMetadata();
|
|
4461
5017
|
}
|
|
4462
5018
|
/**
|
|
4463
5019
|
* Constructs a data frame from a dictionary.
|