qvdjs 0.9.0 → 0.9.2

package/dist/index.js

@@ -1,5 +1,5 @@
-import path2 from 'path';
-import fs2 from 'fs';
+import fs from 'fs';
+import path from 'path';
 import crypto from 'crypto';
 import xml2 from 'xml2js';
 import assert from 'assert';
@@ -250,25 +250,101 @@ var init_QvdSymbol = __esm({
     };
   }
 });
+function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
+  const isCaseInsensitiveFS = process.platform === "win32";
+  const base = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
+  const target = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
+  const relative = path.relative(base, target);
+  if (relative === "") {
+    return true;
+  }
+  if (path.isAbsolute(relative)) {
+    return false;
+  }
+  return relative !== ".." && !relative.startsWith(`..${path.sep}`);
+}
+function resolveDeepestExisting(target) {
+  let current = target;
+  for (; ; ) {
+    try {
+      return fs.realpathSync(current);
+    } catch (error) {
+      const code = (
+        /** @type {{code?: string}} */
+        error?.code
+      );
+      if (code !== "ENOENT" && code !== "ENOTDIR") {
+        return null;
+      }
+      const parent = path.dirname(current);
+      if (parent === current) {
+        return null;
+      }
+      current = parent;
+    }
+  }
+}
+function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
+  let baseStat;
+  try {
+    baseStat = fs.statSync(fs.realpathSync(resolvedBaseDir));
+  } catch {
+    return null;
+  }
+  let current = resolveDeepestExisting(resolvedPath);
+  if (current === null) {
+    return null;
+  }
+  for (; ; ) {
+    let stat;
+    try {
+      stat = fs.statSync(current);
+    } catch {
+      return null;
+    }
+    if (stat.dev === baseStat.dev && stat.ino === baseStat.ino) {
+      return true;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) {
+      return false;
+    }
+    current = parent;
+  }
+}
 function validatePath(filePath, allowedDir) {
+  if (typeof filePath !== "string" || filePath.length === 0) {
+    throw new QvdValidationError("filePath must be a non-empty string", {
+      provided: filePath,
+      type: typeof filePath
+    });
+  }
+  if (allowedDir !== void 0 && allowedDir !== null && typeof allowedDir !== "string") {
+    throw new QvdValidationError("allowedDir must be a string, null or undefined", {
+      provided: allowedDir,
+      type: typeof allowedDir
+    });
+  }
   if (filePath.includes("\0")) {
     throw new QvdSecurityError("Path traversal detected: Null byte in path", {
       path: filePath,
       reason: "null_byte"
     });
   }
+  const resolvedPath = path.resolve(filePath);
   const baseDir = allowedDir || process.cwd();
-  const resolvedBaseDir = path2.resolve(baseDir);
-  const resolvedPath = path2.resolve(filePath);
-  const isCaseInsensitiveFS = process.platform === "win32" || process.platform === "darwin";
-  const baseForComparison = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
-  const pathForComparison = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
-  if (!pathForComparison.startsWith(baseForComparison + path2.sep) && pathForComparison !== baseForComparison) {
+  const resolvedBaseDir = path.resolve(baseDir);
+  const onDisk = isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath);
+  const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk;
+  if (!contained) {
     throw new QvdSecurityError("Path traversal detected: Access denied", {
       path: filePath,
       resolvedPath,
       allowedDir: resolvedBaseDir,
-      reason: "outside_allowed_directory"
+      reason: "outside_allowed_directory",
+      // Says which check refused, so a rejection of a path that looks contained is traceable to
+      // a symlink or a case difference rather than looking like a bug.
+      check: onDisk === null ? "lexical" : "filesystem"
     });
   }
   return resolvedPath;
@@ -297,7 +373,10 @@ var init_QvdFileWriter = __esm({
        * @param {QvdDataFrame} df The data frame to write to the QVD file.
        * @param {Object} [options={}] Options for the writer.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to current working directory.
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
+       *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
+       *   empty value falls back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function.
        */
       constructor(filePath, df, options = {}) {
@@ -343,7 +422,7 @@ var init_QvdFileWriter = __esm({
         const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
         let fd;
         try {
-          fd = await fs2.promises.open(this._path, "w");
+          fd = await fs.promises.open(this._path, "w");
           await fd.write(headerBuffer, 0, headerBuffer.length, 0);
           await fd.write(this._symbolBuffer, 0, this._symbolBuffer.length, headerBuffer.length);
           await fd.write(this._indexBuffer, 0, this._indexBuffer.length, headerBuffer.length + this._symbolBuffer.length);
@@ -369,7 +448,7 @@ var init_QvdFileWriter = __esm({
           SourceFileUtcTime: existingMetadata.SourceFileUtcTime || "",
           SourceFileSize: existingMetadata.SourceFileSize || -1,
           StaleUtcTime: existingMetadata.StaleUtcTime || "",
-          TableName: existingMetadata.TableName || path2.basename(this._path, path2.extname(this._path)),
+          TableName: existingMetadata.TableName || path.basename(this._path, path.extname(this._path)),
           Compression: existingMetadata.Compression || "",
           Comment: existingMetadata.Comment || "",
           EncryptionInfo: existingMetadata.EncryptionInfo || "",
@@ -389,7 +468,7 @@ var init_QvdFileWriter = __esm({
           SourceFileUtcTime: "",
           SourceFileSize: -1,
           StaleUtcTime: "",
-          TableName: path2.basename(this._path, path2.extname(this._path)),
+          TableName: path.basename(this._path, path.extname(this._path)),
           Compression: "",
           Comment: "",
           EncryptionInfo: "",
@@ -559,21 +638,22 @@ var init_QvdFileWriter = __esm({
             this._emitProgress("index-table", processedRows, numRows);
           }
         });
-        this._df.columns.forEach((column) => {
-          const bitOffset = this._indexTableMetadata?.slice(0, this._df.columns.indexOf(column)).reduce((sum, metadata) => sum + metadata[1], 0);
+        this._df.columns.forEach((column, columnIndex) => {
+          const bitOffset = this._indexTableMetadata?.slice(0, columnIndex).reduce((sum, metadata) => sum + metadata[1], 0);
           assert(this._indexTable, "The QVD file header has not been parsed.");
-          const bitWidth = this._indexTable.length > 0 ? Math.max(
-            ...this._indexTable.map(
-              (indices) => indices[this._df.columns.indexOf(column)].length
-            )
-          ) : 0;
-          const fieldContainsNull = this._symbolTableMetadata?.[this._df.columns.indexOf(column)][2];
+          const fieldContainsNull = this._symbolTableMetadata?.[columnIndex][2];
           const bias = fieldContainsNull ? -2 : 0;
+          const symbolCount = this._symbolTable?.[columnIndex].length ?? 0;
+          const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + (fieldContainsNull ? 2 : 0);
+          const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
           this._indexTableMetadata?.push([bitOffset, bitWidth, bias]);
           this._indexTable.forEach((indices) => {
-            const bitString = indices[this._df.columns.indexOf(column)];
-            const paddedBitString = bitString.padStart(bitWidth, "0");
-            indices[this._df.columns.indexOf(column)] = paddedBitString;
+            const bitString = indices[columnIndex];
+            assert(
+              bitString === "0" || bitString.length <= bitWidth,
+              `Stored index for column '${column}' needs ${bitString.length} bits, but BitWidth was derived as ${bitWidth} from ${symbolCount} symbols. The symbol table and the index table are out of sync.`
+            );
+            indices[columnIndex] = bitWidth === 0 ? "" : bitString.padStart(bitWidth, "0");
           });
         });
         this._indexBuffer = Buffer.concat(
@@ -581,6 +661,9 @@ var init_QvdFileWriter = __esm({
           this._indexTable.map((indices) => {
             indices.reverse();
             const bits = indices.join("");
+            if (bits.length === 0) {
+              return Buffer.from(Uint8Array.from([0]));
+            }
             const paddingWidth = (8 - bits.length % 8) % 8;
             const paddedBits = bits.padStart(bits.length + paddingWidth, "0");
             const bytes = paddedBits.match(/.{1,8}/g)?.map((byte) => parseInt(byte, 2));
@@ -793,7 +876,7 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
     });
   }
 }
-function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath) {
+function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
   if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
     throw new QvdCorruptedError("Invalid record byte size", {
       recordSize,
@@ -839,16 +922,40 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
       stage: "parseIndexTable"
     });
   }
-  const maxReasonableOverage = 100 * 1024 * 1024;
-  if (indexTableLength > bufferLength + maxReasonableOverage) {
-    throw new QvdCorruptedError("Index table length unreasonably large", {
+  if (indexTableLength !== totalRows * recordSize) {
+    throw new QvdCorruptedError("Index table length inconsistent with record count", {
       indexTableLength,
-      bufferSize: bufferLength,
+      expectedLength: totalRows * recordSize,
+      totalRows,
+      recordSize,
       file: filePath,
       stage: "parseIndexTable"
     });
   }
+  if (fileSize !== null) {
+    if (indexTableOffset + indexTableLength > fileSize) {
+      throw new QvdCorruptedError("Index table extends beyond the end of the file", {
+        indexTableOffset,
+        indexTableLength,
+        fileSize,
+        file: filePath,
+        stage: "parseIndexTable"
+      });
+    }
+  }
   const requiredIndexBytes = rowsToLoad * recordSize;
+  if (indexTableOffset + requiredIndexBytes > bufferLength) {
+    throw new QvdCorruptedError("Index table truncated", {
+      indexTableOffset,
+      requiredBytes: requiredIndexBytes,
+      availableBytes: Math.max(0, bufferLength - indexTableOffset),
+      rowsToLoad,
+      recordSize,
+      bufferSize: bufferLength,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   if (indexTableLength < requiredIndexBytes) {
     throw new QvdCorruptedError("Index table length smaller than required", {
       indexTableLength,
@@ -1159,7 +1266,7 @@ var QvdFileReader_exports = {};
 __export(QvdFileReader_exports, {
   QvdFileReader: () => QvdFileReader
 });
-var QvdFileReader;
+var MAX_HEADER_SIZE, READ_CHUNK_SIZE, QvdFileReader;
 var init_QvdFileReader = __esm({
   "src/QvdFileReader.js"() {
     init_QvdDataFrame();
@@ -1169,6 +1276,8 @@ var init_QvdFileReader = __esm({
     init_memoryUtils();
     init_validationUtils();
     init_symbolParser();
+    MAX_HEADER_SIZE = 16 * 1024 * 1024;
+    READ_CHUNK_SIZE = 512 * 1024 * 1024;
     QvdFileReader = class {
       /**
        * Constructs a new QVD file parser.
@@ -1176,7 +1285,10 @@ var init_QvdFileReader = __esm({
        * @param {string} filePath The path to the QVD file to load.
        * @param {Object} [options={}] Options for the reader.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to current working directory.
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
+       *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
+       *   empty value falls back to the working directory rather than removing the restriction.
        * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
        *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
        *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
@@ -1192,6 +1304,7 @@ var init_QvdFileReader = __esm({
         this._header = null;
         this._symbolTable = null;
         this._indexTable = null;
+        this._fileSize = null;
       }
       /**
        * Reads the binary data of the QVD file.
@@ -1222,27 +1335,50 @@ var init_QvdFileReader = __esm({
        */
       async _readData(maxRows = null) {
         if (maxRows === null) {
-          this._buffer = await fs2.promises.readFile(this._path);
+          this._buffer = await fs.promises.readFile(this._path);
+          this._fileSize = this._buffer.length;
           return;
         }
         const HEADER_DELIMITER = "\r\n\0";
         const CHUNK_SIZE = 64 * 1024;
-        const stream = fs2.createReadStream(this._path, {
+        const stream = fs.createReadStream(this._path, {
           highWaterMark: CHUNK_SIZE
         });
+        const headerChunks = [];
+        let headerBytes = 0;
+        let tail = Buffer.alloc(0);
         let headerBuffer = Buffer.alloc(0);
         let headerDelimiterIndex = -1;
         try {
           for await (const chunk of stream) {
-            headerBuffer = Buffer.concat([headerBuffer, chunk]);
-            headerDelimiterIndex = headerBuffer.indexOf(HEADER_DELIMITER);
-            if (headerDelimiterIndex !== -1) {
+            const chunkStart = headerBytes;
+            const searchBuffer = tail.length > 0 ? Buffer.concat([tail, chunk]) : chunk;
+            const foundInSearch = searchBuffer.indexOf(HEADER_DELIMITER);
+            headerChunks.push(chunk);
+            headerBytes += chunk.length;
+            if (foundInSearch !== -1) {
+              headerDelimiterIndex = chunkStart - tail.length + foundInSearch;
               stream.destroy();
               break;
             }
+            tail = Buffer.from(searchBuffer.subarray(-(HEADER_DELIMITER.length - 1)));
+            if (headerBytes > MAX_HEADER_SIZE) {
+              stream.destroy();
+              throw new QvdCorruptedError(
+                `The XML header delimiter was not found within the first ${MAX_HEADER_SIZE / (1024 * 1024)}MB of the file.`,
+                {
+                  file: this._path,
+                  bytesSearched: headerBytes,
+                  maxHeaderSize: MAX_HEADER_SIZE,
+                  stage: "readData"
+                }
+              );
+            }
           }
         } catch (error) {
-          if (error && typeof error === "object" && "code" in error && error.code !== "ERR_STREAM_PREMATURE_CLOSE") {
+          const isExpectedEarlyClose = headerDelimiterIndex !== -1 && error !== null && typeof error === "object" && /** @type {{code?: unknown}} */
+          error.code === "ERR_STREAM_PREMATURE_CLOSE";
+          if (!isExpectedEarlyClose) {
             throw error;
           }
         }
@@ -1255,6 +1391,7 @@ var init_QvdFileReader = __esm({
             }
           );
         }
+        headerBuffer = Buffer.concat(headerChunks);
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
         const headerObj = await xml2.parseStringPromise(headerXml, { explicitArray: false });
@@ -1271,12 +1408,50 @@ var init_QvdFileReader = __esm({
         const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
         const rowsToLoad = Math.min(maxRows, totalRows);
         validateSymbolTableSizeEarly(symbolTableLength, this._path);
+        for (const [name, value] of [
+          ["Offset", symbolTableLength],
+          ["RecordByteSize", recordSize],
+          ["NoOfRecords", totalRows]
+        ]) {
+          if (!Number.isSafeInteger(value) || Number(value) < 0) {
+            throw new QvdCorruptedError(`Invalid header value: ${name}`, {
+              name,
+              value,
+              file: this._path,
+              stage: "readData"
+            });
+          }
+        }
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
-        const fd = await fs2.promises.open(this._path, "r");
+        const fd = await fs.promises.open(this._path, "r");
         try {
-          this._buffer = Buffer.allocUnsafe(totalBytesToRead);
-          await fd.read(this._buffer, 0, totalBytesToRead, 0);
+          const { size: fileSize } = await fd.stat();
+          this._fileSize = fileSize;
+          if (totalBytesToRead > fileSize) {
+            throw new QvdCorruptedError("The file is shorter than its header claims.", {
+              file: this._path,
+              fileSize,
+              requiredBytes: totalBytesToRead,
+              stage: "readData"
+            });
+          }
+          this._buffer = Buffer.alloc(totalBytesToRead);
+          let position = 0;
+          while (position < totalBytesToRead) {
+            const length = Math.min(READ_CHUNK_SIZE, totalBytesToRead - position);
+            const { bytesRead } = await fd.read(this._buffer, position, length, position);
+            if (bytesRead === 0) {
+              throw new QvdCorruptedError("Unexpected end of file while reading QVD data.", {
+                file: this._path,
+                fileSize,
+                bytesRead: position,
+                requiredBytes: totalBytesToRead,
+                stage: "readData"
+              });
+            }
+            position += bytesRead;
+          }
         } finally {
           await fd.close();
         }
@@ -1378,7 +1553,10 @@ var init_QvdFileReader = __esm({
             } else {
               symbolIndex = this._convertBitsToInt32(mask.slice(bitOffset, bitOffset + bitWidth));
             }
-            symbolIndex -= bias;
+            symbolIndex += bias;
+            if (symbolIndex < 0) {
+              return;
+            }
             const fieldName = field["FieldName"];
             symbolUsage.get(fieldName)?.add(symbolIndex);
           });
@@ -1483,7 +1661,8 @@ var init_QvdFileReader = __esm({
           this._indexTableOffset,
           this._buffer.length,
           rowsToLoad,
-          this._path
+          this._path,
+          this._fileSize
         );
         const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
         for (const field of fields) {
@@ -1519,14 +1698,33 @@ var init_QvdFileReader = __esm({
           });
           this._indexTable.push(symbolIndices);
         }
+        if (this._indexTable.length !== rowsToLoad) {
+          throw new QvdCorruptedError("Index table contains fewer records than expected", {
+            expectedRows: rowsToLoad,
+            actualRows: this._indexTable.length,
+            totalRows,
+            recordSize,
+            file: this._path,
+            stage: "parseIndexTable"
+          });
+        }
       }
       /**
        * Loads the QVD file into memory and parses it.
        *
        * @param {number|null} maxRows The maximum number of rows to load. If null, all rows are loaded.
+       *   Must be a non-negative integer when given.
+       * @throws {QvdValidationError} If maxRows is neither null nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The loaded QVD file.
        */
       async load(maxRows = null) {
+        if (maxRows !== null && (typeof maxRows !== "number" || !Number.isInteger(maxRows) || maxRows < 0)) {
+          throw new QvdValidationError("maxRows must be a non-negative integer, or null to load all rows", {
+            provided: maxRows,
+            type: typeof maxRows,
+            file: this._path
+          });
+        }
         await this._readData(maxRows);
         await this._parseHeader();
         let symbolsToKeep = null;
@@ -1561,7 +1759,7 @@ var init_QvdFileReader = __esm({
             const symbol = this._symbolTable?.[fieldIndex]?.[symbolIndex];
             const value = symbol?.toPrimaryValue();
             if (typeof value === "string") {
-              if (!isNaN(Number(value))) {
+              if (value.trim() !== "" && !isNaN(Number(value))) {
                 return Number(value);
               }
             }
@@ -1971,7 +2169,11 @@ var init_QvdDataFrame = __esm({
        *
        * @param {string} path The path to the QVD file.
        * @param {Object} [options] Optional writing options.
-       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
+       *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
+       *   back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function that receives progress updates during write operations.
        */
       async toQvd(path3, options = {}) {
@@ -1987,11 +2189,17 @@ var init_QvdDataFrame = __esm({
        *
        * @param {string} path The path to the QVD file.
        * @param {Object} [options] Optional loading options.
-       * @param {number|null} [options.maxRows] The maximum number of rows to load. If not specified, all rows are loaded.
-       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path must be within this directory.
+       * @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
+       *   integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
+       * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
+       *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
+       *   back to the working directory rather than removing the restriction.
        * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
        *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
        *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
+       * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
       static async fromQvd(path3, options = {}) {