qvdjs 0.9.0 → 0.9.2

package/dist/index.cjs

@@ -1,7 +1,7 @@
 'use strict';
 
-var path2 = require('path');
-var fs2 = require('fs');
+var fs = require('fs');
+var path = require('path');
 var crypto = require('crypto');
 var xml2 = require('xml2js');
 var assert = require('assert');
@@ -10,8 +10,8 @@ var v8 = require('v8');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
-var path2__default = /*#__PURE__*/_interopDefault(path2);
-var fs2__default = /*#__PURE__*/_interopDefault(fs2);
+var fs__default = /*#__PURE__*/_interopDefault(fs);
+var path__default = /*#__PURE__*/_interopDefault(path);
 var crypto__default = /*#__PURE__*/_interopDefault(crypto);
 var xml2__default = /*#__PURE__*/_interopDefault(xml2);
 var assert__default = /*#__PURE__*/_interopDefault(assert);
@@ -262,25 +262,101 @@ var init_QvdSymbol = __esm({
     };
   }
 });
+function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
+  const isCaseInsensitiveFS = process.platform === "win32";
+  const base = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
+  const target = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
+  const relative = path__default.default.relative(base, target);
+  if (relative === "") {
+    return true;
+  }
+  if (path__default.default.isAbsolute(relative)) {
+    return false;
+  }
+  return relative !== ".." && !relative.startsWith(`..${path__default.default.sep}`);
+}
+function resolveDeepestExisting(target) {
+  let current = target;
+  for (; ; ) {
+    try {
+      return fs__default.default.realpathSync(current);
+    } catch (error) {
+      const code = (
+        /** @type {{code?: string}} */
+        error?.code
+      );
+      if (code !== "ENOENT" && code !== "ENOTDIR") {
+        return null;
+      }
+      const parent = path__default.default.dirname(current);
+      if (parent === current) {
+        return null;
+      }
+      current = parent;
+    }
+  }
+}
+function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
+  let baseStat;
+  try {
+    baseStat = fs__default.default.statSync(fs__default.default.realpathSync(resolvedBaseDir));
+  } catch {
+    return null;
+  }
+  let current = resolveDeepestExisting(resolvedPath);
+  if (current === null) {
+    return null;
+  }
+  for (; ; ) {
+    let stat;
+    try {
+      stat = fs__default.default.statSync(current);
+    } catch {
+      return null;
+    }
+    if (stat.dev === baseStat.dev && stat.ino === baseStat.ino) {
+      return true;
+    }
+    const parent = path__default.default.dirname(current);
+    if (parent === current) {
+      return false;
+    }
+    current = parent;
+  }
+}
 function validatePath(filePath, allowedDir) {
+  if (typeof filePath !== "string" || filePath.length === 0) {
+    throw new exports.QvdValidationError("filePath must be a non-empty string", {
+      provided: filePath,
+      type: typeof filePath
+    });
+  }
+  if (allowedDir !== void 0 && allowedDir !== null && typeof allowedDir !== "string") {
+    throw new exports.QvdValidationError("allowedDir must be a string, null or undefined", {
+      provided: allowedDir,
+      type: typeof allowedDir
+    });
+  }
   if (filePath.includes("\0")) {
     throw new exports.QvdSecurityError("Path traversal detected: Null byte in path", {
       path: filePath,
       reason: "null_byte"
     });
   }
+  const resolvedPath = path__default.default.resolve(filePath);
   const baseDir = allowedDir || process.cwd();
-  const resolvedBaseDir = path2__default.default.resolve(baseDir);
-  const resolvedPath = path2__default.default.resolve(filePath);
-  const isCaseInsensitiveFS = process.platform === "win32" || process.platform === "darwin";
-  const baseForComparison = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
-  const pathForComparison = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
-  if (!pathForComparison.startsWith(baseForComparison + path2__default.default.sep) && pathForComparison !== baseForComparison) {
+  const resolvedBaseDir = path__default.default.resolve(baseDir);
+  const onDisk = isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath);
+  const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk;
+  if (!contained) {
     throw new exports.QvdSecurityError("Path traversal detected: Access denied", {
       path: filePath,
       resolvedPath,
       allowedDir: resolvedBaseDir,
-      reason: "outside_allowed_directory"
+      reason: "outside_allowed_directory",
+      // Says which check refused, so a rejection of a path that looks contained is traceable to
+      // a symlink or a case difference rather than looking like a bug.
+      check: onDisk === null ? "lexical" : "filesystem"
     });
   }
   return resolvedPath;
@@ -309,7 +385,10 @@ var init_QvdFileWriter = __esm({
        * @param {QvdDataFrame} df The data frame to write to the QVD file.
        * @param {Object} [options={}] Options for the writer.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to current working directory.
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
+       *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
+       *   empty value falls back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function.
        */
       constructor(filePath, df, options = {}) {
@@ -355,7 +434,7 @@ var init_QvdFileWriter = __esm({
         const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
         let fd;
         try {
-          fd = await fs2__default.default.promises.open(this._path, "w");
+          fd = await fs__default.default.promises.open(this._path, "w");
           await fd.write(headerBuffer, 0, headerBuffer.length, 0);
           await fd.write(this._symbolBuffer, 0, this._symbolBuffer.length, headerBuffer.length);
           await fd.write(this._indexBuffer, 0, this._indexBuffer.length, headerBuffer.length + this._symbolBuffer.length);
@@ -381,7 +460,7 @@ var init_QvdFileWriter = __esm({
           SourceFileUtcTime: existingMetadata.SourceFileUtcTime || "",
           SourceFileSize: existingMetadata.SourceFileSize || -1,
           StaleUtcTime: existingMetadata.StaleUtcTime || "",
-          TableName: existingMetadata.TableName || path2__default.default.basename(this._path, path2__default.default.extname(this._path)),
+          TableName: existingMetadata.TableName || path__default.default.basename(this._path, path__default.default.extname(this._path)),
           Compression: existingMetadata.Compression || "",
           Comment: existingMetadata.Comment || "",
           EncryptionInfo: existingMetadata.EncryptionInfo || "",
@@ -401,7 +480,7 @@ var init_QvdFileWriter = __esm({
           SourceFileUtcTime: "",
           SourceFileSize: -1,
           StaleUtcTime: "",
-          TableName: path2__default.default.basename(this._path, path2__default.default.extname(this._path)),
+          TableName: path__default.default.basename(this._path, path__default.default.extname(this._path)),
           Compression: "",
           Comment: "",
           EncryptionInfo: "",
@@ -571,21 +650,22 @@ var init_QvdFileWriter = __esm({
             this._emitProgress("index-table", processedRows, numRows);
           }
         });
-        this._df.columns.forEach((column) => {
-          const bitOffset = this._indexTableMetadata?.slice(0, this._df.columns.indexOf(column)).reduce((sum, metadata) => sum + metadata[1], 0);
+        this._df.columns.forEach((column, columnIndex) => {
+          const bitOffset = this._indexTableMetadata?.slice(0, columnIndex).reduce((sum, metadata) => sum + metadata[1], 0);
           assert__default.default(this._indexTable, "The QVD file header has not been parsed.");
-          const bitWidth = this._indexTable.length > 0 ? Math.max(
-            ...this._indexTable.map(
-              (indices) => indices[this._df.columns.indexOf(column)].length
-            )
-          ) : 0;
-          const fieldContainsNull = this._symbolTableMetadata?.[this._df.columns.indexOf(column)][2];
+          const fieldContainsNull = this._symbolTableMetadata?.[columnIndex][2];
           const bias = fieldContainsNull ? -2 : 0;
+          const symbolCount = this._symbolTable?.[columnIndex].length ?? 0;
+          const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + (fieldContainsNull ? 2 : 0);
+          const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
           this._indexTableMetadata?.push([bitOffset, bitWidth, bias]);
           this._indexTable.forEach((indices) => {
-            const bitString = indices[this._df.columns.indexOf(column)];
-            const paddedBitString = bitString.padStart(bitWidth, "0");
-            indices[this._df.columns.indexOf(column)] = paddedBitString;
+            const bitString = indices[columnIndex];
+            assert__default.default(
+              bitString === "0" || bitString.length <= bitWidth,
+              `Stored index for column '${column}' needs ${bitString.length} bits, but BitWidth was derived as ${bitWidth} from ${symbolCount} symbols. The symbol table and the index table are out of sync.`
+            );
+            indices[columnIndex] = bitWidth === 0 ? "" : bitString.padStart(bitWidth, "0");
           });
         });
         this._indexBuffer = Buffer.concat(
@@ -593,6 +673,9 @@ var init_QvdFileWriter = __esm({
           this._indexTable.map((indices) => {
             indices.reverse();
             const bits = indices.join("");
+            if (bits.length === 0) {
+              return Buffer.from(Uint8Array.from([0]));
+            }
             const paddingWidth = (8 - bits.length % 8) % 8;
             const paddedBits = bits.padStart(bits.length + paddingWidth, "0");
             const bytes = paddedBits.match(/.{1,8}/g)?.map((byte) => parseInt(byte, 2));
@@ -805,7 +888,7 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
     });
   }
 }
-function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath) {
+function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
   if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
     throw new exports.QvdCorruptedError("Invalid record byte size", {
       recordSize,
@@ -851,16 +934,40 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
       stage: "parseIndexTable"
     });
   }
-  const maxReasonableOverage = 100 * 1024 * 1024;
-  if (indexTableLength > bufferLength + maxReasonableOverage) {
-    throw new exports.QvdCorruptedError("Index table length unreasonably large", {
+  if (indexTableLength !== totalRows * recordSize) {
+    throw new exports.QvdCorruptedError("Index table length inconsistent with record count", {
       indexTableLength,
-      bufferSize: bufferLength,
+      expectedLength: totalRows * recordSize,
+      totalRows,
+      recordSize,
       file: filePath,
       stage: "parseIndexTable"
     });
   }
+  if (fileSize !== null) {
+    if (indexTableOffset + indexTableLength > fileSize) {
+      throw new exports.QvdCorruptedError("Index table extends beyond the end of the file", {
+        indexTableOffset,
+        indexTableLength,
+        fileSize,
+        file: filePath,
+        stage: "parseIndexTable"
+      });
+    }
+  }
   const requiredIndexBytes = rowsToLoad * recordSize;
+  if (indexTableOffset + requiredIndexBytes > bufferLength) {
+    throw new exports.QvdCorruptedError("Index table truncated", {
+      indexTableOffset,
+      requiredBytes: requiredIndexBytes,
+      availableBytes: Math.max(0, bufferLength - indexTableOffset),
+      rowsToLoad,
+      recordSize,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   if (indexTableLength < requiredIndexBytes) {
     throw new exports.QvdCorruptedError("Index table length smaller than required", {
       indexTableLength,
@@ -1171,7 +1278,7 @@ var QvdFileReader_exports = {};
 __export(QvdFileReader_exports, {
   QvdFileReader: () => exports.QvdFileReader
 });
-exports.QvdFileReader = void 0;
+var MAX_HEADER_SIZE, READ_CHUNK_SIZE; exports.QvdFileReader = void 0;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
     init_QvdDataFrame();
@@ -1181,6 +1288,8 @@ var init_QvdFileReader = __esm({
     init_memoryUtils();
     init_validationUtils();
     init_symbolParser();
+    MAX_HEADER_SIZE = 16 * 1024 * 1024;
+    READ_CHUNK_SIZE = 512 * 1024 * 1024;
     exports.QvdFileReader = class {
       /**
        * Constructs a new QVD file parser.
@@ -1188,7 +1297,10 @@ var init_QvdFileReader = __esm({
        * @param {string} filePath The path to the QVD file to load.
        * @param {Object} [options={}] Options for the reader.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to current working directory.
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
+       *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
+       *   empty value falls back to the working directory rather than removing the restriction.
        * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
        *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
        *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
@@ -1204,6 +1316,7 @@ var init_QvdFileReader = __esm({
         this._header = null;
         this._symbolTable = null;
         this._indexTable = null;
+        this._fileSize = null;
       }
       /**
        * Reads the binary data of the QVD file.
@@ -1234,27 +1347,50 @@ var init_QvdFileReader = __esm({
        */
       async _readData(maxRows = null) {
         if (maxRows === null) {
-          this._buffer = await fs2__default.default.promises.readFile(this._path);
+          this._buffer = await fs__default.default.promises.readFile(this._path);
+          this._fileSize = this._buffer.length;
           return;
         }
         const HEADER_DELIMITER = "\r\n\0";
         const CHUNK_SIZE = 64 * 1024;
-        const stream = fs2__default.default.createReadStream(this._path, {
+        const stream = fs__default.default.createReadStream(this._path, {
           highWaterMark: CHUNK_SIZE
         });
+        const headerChunks = [];
+        let headerBytes = 0;
+        let tail = Buffer.alloc(0);
         let headerBuffer = Buffer.alloc(0);
         let headerDelimiterIndex = -1;
         try {
           for await (const chunk of stream) {
-            headerBuffer = Buffer.concat([headerBuffer, chunk]);
-            headerDelimiterIndex = headerBuffer.indexOf(HEADER_DELIMITER);
-            if (headerDelimiterIndex !== -1) {
+            const chunkStart = headerBytes;
+            const searchBuffer = tail.length > 0 ? Buffer.concat([tail, chunk]) : chunk;
+            const foundInSearch = searchBuffer.indexOf(HEADER_DELIMITER);
+            headerChunks.push(chunk);
+            headerBytes += chunk.length;
+            if (foundInSearch !== -1) {
+              headerDelimiterIndex = chunkStart - tail.length + foundInSearch;
               stream.destroy();
               break;
             }
+            tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
+            if (headerBytes > MAX_HEADER_SIZE) {
+              stream.destroy();
+              throw new exports.QvdCorruptedError(
+                `The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
+                {
+                  file: this._path,
+                  bytesSearched: headerBytes,
+                  maxHeaderSize: MAX_HEADER_SIZE,
+                  stage: "readData"
+                }
+              );
+            }
           }
         } catch (error) {
-          if (error && typeof error === "object" && "code" in error && error.code !== "ERR_STREAM_PREMATURE_CLOSE") {
+          const isExpectedEarlyClose = headerDelimiterIndex !== -1 && error !== null && typeof error === "object" && /** @type {{code?: unknown}} */
+          error.code === "ERR_STREAM_PREMATURE_CLOSE";
+          if (!isExpectedEarlyClose) {
             throw error;
           }
         }
@@ -1267,6 +1403,7 @@ var init_QvdFileReader = __esm({
             }
           );
         }
+        headerBuffer = Buffer.concat(headerChunks);
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
         const headerObj = await xml2__default.default.parseStringPromise(headerXml, { explicitArray: false });
@@ -1283,12 +1420,50 @@ var init_QvdFileReader = __esm({
         const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
         const rowsToLoad = Math.min(maxRows, totalRows);
         validateSymbolTableSizeEarly(symbolTableLength, this._path);
+        for (const [name, value] of [
+          ["Offset", symbolTableLength],
+          ["RecordByteSize", recordSize],
+          ["NoOfRecords", totalRows]
+        ]) {
+          if (!Number.isSafeInteger(value) || Number(value) < 0) {
+            throw new exports.QvdCorruptedError(`Invalid header value: ${name}`, {
+              name,
+              value,
+              file: this._path,
+              stage: "readData"
+            });
+          }
+        }
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
-        const fd = await fs2__default.default.promises.open(this._path, "r");
+        const fd = await fs__default.default.promises.open(this._path, "r");
         try {
-          this._buffer = Buffer.allocUnsafe(totalBytesToRead);
-          await fd.read(this._buffer, 0, totalBytesToRead, 0);
+          const { size: fileSize } = await fd.stat();
+          this._fileSize = fileSize;
+          if (totalBytesToRead > fileSize) {
+            throw new exports.QvdCorruptedError("The file is shorter than its header claims.", {
+              file: this._path,
+              fileSize,
+              requiredBytes: totalBytesToRead,
+              stage: "readData"
+            });
+          }
+          this._buffer = Buffer.alloc(totalBytesToRead);
+          let position = 0;
+          while (position < totalBytesToRead) {
+            const length = Math.min(READ_CHUNK_SIZE, totalBytesToRead - position);
+            const { bytesRead } = await fd.read(this._buffer, position, length, position);
+            if (bytesRead === 0) {
+              throw new exports.QvdCorruptedError("Unexpected end of file while reading QVD data.", {
+                file: this._path,
+                fileSize,
+                bytesRead: position,
+                requiredBytes: totalBytesToRead,
+                stage: "readData"
+              });
+            }
+            position += bytesRead;
+          }
         } finally {
           await fd.close();
         }
@@ -1390,7 +1565,10 @@ var init_QvdFileReader = __esm({
             } else {
               symbolIndex = this._convertBitsToInt32(mask.slice(bitOffset, bitOffset + bitWidth));
             }
-            symbolIndex -= bias;
+            symbolIndex += bias;
+            if (symbolIndex < 0) {
+              return;
+            }
             const fieldName = field["FieldName"];
             symbolUsage.get(fieldName)?.add(symbolIndex);
           });
@@ -1495,7 +1673,8 @@ var init_QvdFileReader = __esm({
           this._indexTableOffset,
           this._buffer.length,
           rowsToLoad,
-          this._path
+          this._path,
+          this._fileSize
         );
         const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
         for (const field of fields) {
@@ -1531,14 +1710,33 @@ var init_QvdFileReader = __esm({
           });
           this._indexTable.push(symbolIndices);
         }
+        if (this._indexTable.length !== rowsToLoad) {
+          throw new exports.QvdCorruptedError("Index table contains fewer records than expected", {
+            expectedRows: rowsToLoad,
+            actualRows: this._indexTable.length,
+            totalRows,
+            recordSize,
+            file: this._path,
+            stage: "parseIndexTable"
+          });
+        }
       }
       /**
        * Loads the QVD file into memory and parses it.
        *
        * @param {number|null} maxRows The maximum number of rows to load. If null, all rows are loaded.
+       *   Must be a non-negative integer when given.
+       * @throws {QvdValidationError} If maxRows is neither null nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The loaded QVD file.
        */
       async load(maxRows = null) {
+        if (maxRows !== null && (typeof maxRows !== "number" || !Number.isInteger(maxRows) || maxRows < 0)) {
+          throw new exports.QvdValidationError("maxRows must be a non-negative integer, or null to load all rows", {
+            provided: maxRows,
+            type: typeof maxRows,
+            file: this._path
+          });
+        }
         await this._readData(maxRows);
         await this._parseHeader();
         let symbolsToKeep = null;
@@ -1573,7 +1771,7 @@ var init_QvdFileReader = __esm({
             const symbol = this._symbolTable?.[fieldIndex]?.[symbolIndex];
             const value = symbol?.toPrimaryValue();
             if (typeof value === "string") {
-              if (!isNaN(Number(value))) {
+              if (value.trim() !== "" && !isNaN(Number(value))) {
                 return Number(value);
               }
             }
@@ -1983,7 +2181,11 @@ var init_QvdDataFrame = __esm({
        *
        * @param {string} path The path to the QVD file.
        * @param {Object} [options] Optional writing options.
-       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
+       *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
+       *   back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function that receives progress updates during write operations.
        */
       async toQvd(path3, options = {}) {
@@ -1999,11 +2201,17 @@ var init_QvdDataFrame = __esm({
        *
        * @param {string} path The path to the QVD file.
        * @param {Object} [options] Optional loading options.
-       * @param {number|null} [options.maxRows] The maximum number of rows to load. If not specified, all rows are loaded.
-       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
+       *   integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
+       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
+       *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
+       *   back to the working directory rather than removing the restriction.
        * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
        *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
        *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
+       * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
       static async fromQvd(path3, options = {}) {