qvdjs 0.9.1 → 0.9.3

package/dist/index.js

@@ -1,5 +1,5 @@
+import fs from 'fs';
 import path from 'path';
-import fs2 from 'fs';
 import crypto from 'crypto';
 import xml2 from 'xml2js';
 import assert from 'assert';
@@ -250,8 +250,8 @@ var init_QvdSymbol = __esm({
     };
   }
 });
-function isWithinDirectory(resolvedBaseDir, resolvedPath) {
-  const isCaseInsensitiveFS = process.platform === "win32" || process.platform === "darwin";
+function isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) {
+  const isCaseInsensitiveFS = process.platform === "win32";
   const base = isCaseInsensitiveFS ? resolvedBaseDir.toLowerCase() : resolvedBaseDir;
   const target = isCaseInsensitiveFS ? resolvedPath.toLowerCase() : resolvedPath;
   const relative = path.relative(base, target);
@@ -263,6 +263,55 @@ function isWithinDirectory(resolvedBaseDir, resolvedPath) {
   }
   return relative !== ".." && !relative.startsWith(`..${path.sep}`);
 }
+function resolveDeepestExisting(target) {
+  let current = target;
+  for (; ; ) {
+    try {
+      return fs.realpathSync(current);
+    } catch (error) {
+      const code = (
+        /** @type {{code?: string}} */
+        error?.code
+      );
+      if (code !== "ENOENT" && code !== "ENOTDIR") {
+        return null;
+      }
+      const parent = path.dirname(current);
+      if (parent === current) {
+        return null;
+      }
+      current = parent;
+    }
+  }
+}
+function isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath) {
+  let baseStat;
+  try {
+    baseStat = fs.statSync(fs.realpathSync(resolvedBaseDir));
+  } catch {
+    return null;
+  }
+  let current = resolveDeepestExisting(resolvedPath);
+  if (current === null) {
+    return null;
+  }
+  for (; ; ) {
+    let stat;
+    try {
+      stat = fs.statSync(current);
+    } catch {
+      return null;
+    }
+    if (stat.dev === baseStat.dev && stat.ino === baseStat.ino) {
+      return true;
+    }
+    const parent = path.dirname(current);
+    if (parent === current) {
+      return false;
+    }
+    current = parent;
+  }
+}
 function validatePath(filePath, allowedDir) {
   if (typeof filePath !== "string" || filePath.length === 0) {
     throw new QvdValidationError("filePath must be a non-empty string", {
@@ -285,12 +334,17 @@ function validatePath(filePath, allowedDir) {
   const resolvedPath = path.resolve(filePath);
   const baseDir = allowedDir || process.cwd();
   const resolvedBaseDir = path.resolve(baseDir);
-  if (!isWithinDirectory(resolvedBaseDir, resolvedPath)) {
+  const onDisk = isWithinDirectoryOnDisk(resolvedBaseDir, resolvedPath);
+  const contained = onDisk === null ? isWithinDirectoryLexically(resolvedBaseDir, resolvedPath) : onDisk;
+  if (!contained) {
     throw new QvdSecurityError("Path traversal detected: Access denied", {
       path: filePath,
       resolvedPath,
       allowedDir: resolvedBaseDir,
-      reason: "outside_allowed_directory"
+      reason: "outside_allowed_directory",
+      // Says which check refused, so a rejection of a path that looks contained is traceable to
+      // a symlink or a case difference rather than looking like a bug.
+      check: onDisk === null ? "lexical" : "filesystem"
     });
   }
   return resolvedPath;
@@ -319,7 +373,8 @@ var init_QvdFileWriter = __esm({
        * @param {QvdDataFrame} df The data frame to write to the QVD file.
        * @param {Object} [options={}] Options for the writer.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to the current working directory. To permit
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
        *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
        *   empty value falls back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function.
@@ -367,7 +422,7 @@ var init_QvdFileWriter = __esm({
         const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
         let fd;
         try {
-          fd = await fs2.promises.open(this._path, "w");
+          fd = await fs.promises.open(this._path, "w");
           await fd.write(headerBuffer, 0, headerBuffer.length, 0);
           await fd.write(this._symbolBuffer, 0, this._symbolBuffer.length, headerBuffer.length);
           await fd.write(this._indexBuffer, 0, this._indexBuffer.length, headerBuffer.length + this._symbolBuffer.length);
@@ -679,6 +734,28 @@ var init_bitUtils = __esm({
 function getHeapLimit() {
   return v8.getHeapStatistics().heap_size_limit;
 }
+function heapLimitIsMeaningful() {
+  return !process.versions.bun && !process.versions.deno;
+}
+function getMemoryBudget() {
+  const candidates = [];
+  if (heapLimitIsMeaningful()) {
+    candidates.push({ source: "V8 heap limit", bytes: getHeapLimit() });
+  }
+  const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
+  if (constrained > 0 && constrained < os.totalmem()) {
+    candidates.push({ source: "container memory limit", bytes: constrained });
+  }
+  if (candidates.length === 0) {
+    candidates.push({ source: "total system memory", bytes: os.totalmem() });
+  }
+  const observed = [{ source: "free memory (os.freemem)", bytes: os.freemem() }];
+  if (typeof process.availableMemory === "function") {
+    observed.push({ source: "available memory", bytes: process.availableMemory() });
+  }
+  const binding = candidates.reduce((lowest, candidate) => candidate.bytes < lowest.bytes ? candidate : lowest);
+  return { bytes: binding.bytes, limitedBy: binding.source, candidates, observed };
+}
 function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
   const FULL_PARSE_OVERHEAD = 6;
   const MINIMAL_OVERHEAD = 0.01;
@@ -695,11 +772,14 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
   if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
     throw new QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
   }
-  const availableMemory = os.freemem();
+  if (safetyFactor === 0) {
+    return;
+  }
+  const budget = getMemoryBudget();
   const heapLimit = getHeapLimit();
+  const availableMemory = budget.bytes;
   const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
-  const effectiveLimit = Math.min(availableMemory, heapLimit);
-  const maxAllowedMemory = effectiveLimit * safetyFactor;
+  const maxAllowedMemory = budget.bytes * safetyFactor;
   if (estimatedMemory > maxAllowedMemory) {
     const safeSymbolPercentage = maxAllowedMemory / (symbolTableSize * 6);
     const safeRowPercentage = Math.pow(safeSymbolPercentage, 2);
@@ -709,9 +789,12 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
     const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
     const heapLimitMB = Math.round(heapLimit / 1024 / 1024);
     const availableRamMB = Math.round(availableMemory / 1024 / 1024);
-    const limitingFactor = heapLimit < availableMemory ? "V8 heap limit" : "available RAM";
+    const limitingFactor = budget.limitedBy;
+    const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
+    const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
+    const advice = limitingFactor === "container memory limit" ? `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).` : `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
     throw new QvdValidationError(
-      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}: ${heapLimitMB}MB heap / ${availableRamMB}MB RAM). Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`,
+      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
       {
         file: filePath,
         symbolTableSize,
@@ -721,6 +804,8 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
         heapLimitMB,
         availableRamMB,
         limitingFactor,
+        memoryBudget: budget.candidates,
+        memoryObserved: budget.observed,
         totalRows,
         maxRows,
         recommendedMaxRows
@@ -1230,17 +1315,26 @@ var init_QvdFileReader = __esm({
        * @param {string} filePath The path to the QVD file to load.
        * @param {Object} [options={}] Options for the reader.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file
-       *   path must be within this directory. Defaults to the current working directory. To permit
+       *   path must be within this directory, with symlinks resolved first, so a link inside it that
+       *   points outside it is rejected. Defaults to the current working directory. To permit
        *   an entire volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or
        *   empty value falls back to the working directory rather than removing the restriction.
-       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines
-       *   what percentage of available memory (or V8 heap limit, whichever is smaller) can be used
-       *   for loading QVD files. Default is 0.3 (30%). Increase for larger heap configurations.
+       * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a
+       *   load may use. The budget is the smallest of the V8 heap limit, any container memory limit,
+       *   and the memory the OS reports as available. Default is 0.3. **Zero disables the memory
+       *   check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
+       *   Bun reports its current heap as its heap limit - and for callers who would rather manage
+       *   memory themselves than trust the estimate.
+       * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes,
+       *   above which a lazy load switches to the two-pass filtering path. The default of 50MB is
+       *   the point where the extra analysis pass pays for itself; lower it to use filtering on
+       *   smaller files, raise it to keep the simpler single-pass read for longer.
        */
       constructor(filePath, options = {}) {
-        const { allowedDir, memorySafetyFactor = 0.3 } = options;
+        const { allowedDir, memorySafetyFactor = 0.3, symbolFilteringThreshold = 50 * 1024 * 1024 } = options;
         this._path = validatePath(filePath, allowedDir);
         this._memorySafetyFactor = memorySafetyFactor;
+        this._symbolFilteringThreshold = symbolFilteringThreshold;
         this._buffer = null;
         this._headerOffset = null;
         this._symbolTableOffset = null;
@@ -1279,13 +1373,13 @@ var init_QvdFileReader = __esm({
        */
       async _readData(maxRows = null) {
         if (maxRows === null) {
-          this._buffer = await fs2.promises.readFile(this._path);
+          this._buffer = await fs.promises.readFile(this._path);
           this._fileSize = this._buffer.length;
           return;
         }
         const HEADER_DELIMITER = "\r\n\0";
         const CHUNK_SIZE = 64 * 1024;
-        const stream = fs2.createReadStream(this._path, {
+        const stream = fs.createReadStream(this._path, {
           highWaterMark: CHUNK_SIZE
         });
         const headerChunks = [];
@@ -1368,7 +1462,7 @@ var init_QvdFileReader = __esm({
         }
         const indexTableBytesToRead = rowsToLoad * recordSize;
         const totalBytesToRead = indexTableOffset + indexTableBytesToRead;
-        const fd = await fs2.promises.open(this._path, "r");
+        const fd = await fs.promises.open(this._path, "r");
         try {
           const { size: fileSize } = await fd.stat();
           this._fileSize = fileSize;
@@ -1672,15 +1766,12 @@ var init_QvdFileReader = __esm({
         await this._readData(maxRows);
         await this._parseHeader();
         let symbolsToKeep = null;
+        let symbolsKept = null;
         if (maxRows !== null && this._header) {
           const symbolTableLength = parseInt(this._header["QvdTableHeader"]["Offset"], 10);
-          const SYMBOL_FILTERING_THRESHOLD = 50 * 1024 * 1024;
-          if (symbolTableLength > SYMBOL_FILTERING_THRESHOLD) {
+          if (symbolTableLength > this._symbolFilteringThreshold) {
             symbolsToKeep = await this._analyzeIndexTableSymbolUsage(maxRows);
-            const totalSymbols = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
-            console.log(
-              `[Phase 2.5 Optimization] Using stream-and-skip parsing: keeping ${totalSymbols} symbols from ${(symbolTableLength / 1024 / 1024).toFixed(1)}MB symbol table`
-            );
+            symbolsKept = Array.from(symbolsToKeep.values()).reduce((sum, set) => sum + set.size, 0);
           }
         }
         await this._parseSymbolTable(symbolsToKeep, maxRows);
@@ -1717,7 +1808,14 @@ var init_QvdFileReader = __esm({
         const columns = fields.map((field) => field["FieldName"]);
         const data = this._indexTable.map((_, index) => getRow(index));
         const metadata = this._header["QvdTableHeader"];
-        return new QvdDataFrame(data, columns, metadata);
+        const loadStats = {
+          symbolTableBytes: parseInt(this._header["QvdTableHeader"]["Offset"], 10),
+          totalRows: parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10),
+          rowsLoaded: data.length,
+          symbolFiltering: symbolsToKeep !== null,
+          symbolsKept
+        };
+        return new QvdDataFrame(data, columns, metadata, loadStats);
       }
     };
   }
@@ -1734,11 +1832,13 @@ var init_QvdDataFrame = __esm({
        * @param {Array<Array<any>>} data The data of the data frame.
        * @param {Array<string>} columns The columns of the data frame.
        * @param {QvdMetadata|null} metadata The metadata from the QVD file header (optional).
+       * @param {QvdLoadStats|null} loadStats Statistics about the read (optional).
        */
-      constructor(data, columns, metadata = null) {
+      constructor(data, columns, metadata = null, loadStats = null) {
         this._data = data;
         this._columns = columns;
         this._metadata = metadata;
+        this._loadStats = loadStats;
       }
       /**
        * Returns the data of the data frame.
@@ -1765,6 +1865,21 @@ var init_QvdDataFrame = __esm({
       get metadata() {
         return this._metadata;
       }
+      /**
+       * Returns statistics about the read that produced this data frame.
+       *
+       * Only a frame returned by fromQvd() carries these; fromDict(), head() and tail() produce
+       * frames that describe no particular read, and report null rather than a stale figure.
+       *
+       * The main use is confirming that a lazy load actually filtered the symbol table:
+       * `symbolFiltering` says whether the two-pass path ran, and `symbolsKept` how many symbols
+       * survived it, which for a small maxRows should be a tiny fraction of the file's total.
+       *
+       * @return {QvdLoadStats|null} Load statistics, or null if this frame did not come from a file.
+       */
+      get loadStats() {
+        return this._loadStats;
+      }
       /**
        * Returns file-level metadata from the QVD header.
        * @return {Object} File-level metadata properties.
@@ -2114,7 +2229,8 @@ var init_QvdDataFrame = __esm({
        * @param {string} path The path to the QVD file.
        * @param {Object} [options] Optional writing options.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
-       *   must be within this directory. Defaults to the current working directory. To permit an entire
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
        *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
        *   back to the working directory rather than removing the restriction.
        * @param {Function} [options.onProgress] Optional progress callback function that receives progress updates during write operations.
@@ -2135,12 +2251,16 @@ var init_QvdDataFrame = __esm({
        * @param {number|null} [options.maxRows] The maximum number of rows to load. Must be a non-negative
        *   integer; if not specified or null, all rows are loaded. Anything else throws a QvdValidationError.
        * @param {string} [options.allowedDir] Optional allowed directory path. If provided, the file path
-       *   must be within this directory. Defaults to the current working directory. To permit an entire
+       *   must be within this directory, with symlinks resolved first, so a link inside it that points
+       *   outside it is rejected. Defaults to the current working directory. To permit an entire
        *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
        *   back to the working directory rather than removing the restriction.
-       * @param {number} [options.memorySafetyFactor=0.3] Memory safety factor (0.0-1.0). Determines what percentage
-       *   of available memory (or V8 heap limit, whichever is smaller) can be used. Default is 0.3 (30%).
-       *   Increase this (e.g., to 0.5 or 0.7) when running with larger heap sizes via --max-old-space-size.
+       * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a load
+       *   may use. The budget is the smallest of the V8 heap limit, any container memory limit, and the
+       *   memory the OS reports as available. Default is 0.3; raise it when running with a larger heap via
+       *   --max-old-space-size. **Zero disables the memory check entirely.**
+       * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
+       *   a lazy load switches to the two-pass filtering path. Defaults to 50MB.
        * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
        * @return {Promise<QvdDataFrame>} The data frame of the QVD file.
        */
@@ -2148,7 +2268,8 @@ var init_QvdDataFrame = __esm({
         const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
         const readerOptions = {
           allowedDir: options.allowedDir,
-          memorySafetyFactor: options.memorySafetyFactor
+          memorySafetyFactor: options.memorySafetyFactor,
+          symbolFilteringThreshold: options.symbolFilteringThreshold
         };
         return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
       }