qvdjs 0.9.4 → 0.10.1

package/dist/index.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import crypto from 'crypto';
 import xml2 from 'xml2js';
-import assert from 'assert';
+import assert2 from 'assert';
 import os from 'os';
 import v8 from 'v8';
 
@@ -355,6 +355,55 @@ var init_validatePath = __esm({
   }
 });
 
+// src/util/bitUtils.js
+function fieldGeometry(bitOffset, bitWidth) {
+  const shift = bitOffset & 7;
+  return {
+    byteStart: bitOffset >>> 3,
+    shift,
+    byteCount: bitWidth === 0 ? 0 : shift + bitWidth + 7 >>> 3
+  };
+}
+function decodeIndexColumn(buffer, recordSize, rowCount, bitOffset, bitWidth, bias, out) {
+  if (bitWidth === 0) {
+    out.fill(bias, 0, rowCount);
+    return out;
+  }
+  const { byteStart, shift, byteCount } = fieldGeometry(bitOffset, bitWidth);
+  const divisor = POW2[shift];
+  const modulus = POW2[bitWidth];
+  let base = byteStart;
+  for (let row = 0; row < rowCount; row++, base += recordSize) {
+    let acc = buffer[base];
+    if (byteCount > 1) acc += buffer[base + 1] * 256;
+    if (byteCount > 2) acc += buffer[base + 2] * 65536;
+    if (byteCount > 3) acc += buffer[base + 3] * 16777216;
+    if (byteCount > 4) acc += buffer[base + 4] * 4294967296;
+    out[row] = Math.floor(acc / divisor) % modulus + bias;
+  }
+  return out;
+}
+function writeBitField(buffer, recordBase, geometry, value) {
+  const { byteStart, shift, byteCount } = geometry;
+  if (byteCount === 0) {
+    return;
+  }
+  const base = recordBase + byteStart;
+  const shifted = value * POW2[shift];
+  buffer[base] |= shifted % 256;
+  if (byteCount > 1) buffer[base + 1] |= Math.floor(shifted / 256) % 256;
+  if (byteCount > 2) buffer[base + 2] |= Math.floor(shifted / 65536) % 256;
+  if (byteCount > 3) buffer[base + 3] |= Math.floor(shifted / 16777216) % 256;
+  if (byteCount > 4) buffer[base + 4] |= Math.floor(shifted / 4294967296) % 256;
+}
+var MAX_BIT_WIDTH, POW2;
+var init_bitUtils = __esm({
+  "src/util/bitUtils.js"() {
+    MAX_BIT_WIDTH = 31;
+    POW2 = Array.from({ length: 41 }, (_, exponent) => 2 ** exponent);
+  }
+});
+
 // src/QvdFileWriter.js
 var QvdFileWriter_exports = {};
 __export(QvdFileWriter_exports, {
@@ -364,7 +413,9 @@ var QvdFileWriter;
 var init_QvdFileWriter = __esm({
   "src/QvdFileWriter.js"() {
     init_QvdSymbol();
+    init_QvdErrors();
     init_validatePath();
+    init_bitUtils();
     QvdFileWriter = class _QvdFileWriter {
       /**
        * Constructs a new QVD file writer.
@@ -389,8 +440,9 @@ var init_QvdFileWriter = __esm({
         this._symbolTable = null;
         this._symbolTableMetadata = null;
         this._indexBuffer = null;
-        this._indexTable = null;
+        this._symbolIndexByValue = null;
         this._indexTableMetadata = null;
+        this._recordCount = 0;
         this._recordByteSize = null;
       }
       /**
@@ -415,9 +467,9 @@ var init_QvdFileWriter = __esm({
        * Writes the data to the QVD file.
        */
       async _writeData() {
-        assert(this._header, "The QVD file header has not been parsed.");
-        assert(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
-        assert(this._indexBuffer, "The QVD file index table has not been parsed.");
+        assert2(this._header, "The QVD file header has not been parsed.");
+        assert2(this._symbolBuffer, "The QVD file symbol table has not been parsed.");
+        assert2(this._indexBuffer, "The QVD file index table has not been parsed.");
         this._emitProgress("write", 0, 1);
         const headerBuffer = Buffer.concat([Buffer.from(this._header, "utf-8"), Buffer.from([0])]);
         let fd;
@@ -512,7 +564,7 @@ var init_QvdFileWriter = __esm({
                 };
               })
             },
-            NoOfRecords: this._indexTable?.length,
+            NoOfRecords: this._recordCount,
             RecordByteSize: this._recordByteSize,
             Offset: this._symbolTableMetadata && this._symbolTableMetadata.length > 0 ? this._symbolTableMetadata[this._symbolTableMetadata.length - 1][0] + this._symbolTableMetadata[this._symbolTableMetadata.length - 1][1] : 0,
             Length: this._indexBuffer?.length
@@ -551,128 +603,133 @@ var init_QvdFileWriter = __esm({
       _buildSymbolTable() {
         this._symbolTable = [];
         this._symbolTableMetadata = [];
-        this._symbolBuffer = Buffer.alloc(0);
-        const numColumns = this._df.columns.length;
-        this._emitProgress("symbol-table", 0, numColumns);
-        const uniqueValuesSets = this._df.columns.map(() => /* @__PURE__ */ new Set());
-        this._df.data.forEach((row) => {
-          this._df.columns.forEach((column, columnIndex) => {
-            const value = row[columnIndex];
-            uniqueValuesSets[columnIndex].add(value);
+        this._symbolIndexByValue = [];
+        if (this._df.columns.length === 0) {
+          throw new QvdValidationError("A QVD file must have at least one field", {
+            file: this._path,
+            stage: "buildSymbolTable"
           });
-        });
-        this._df.columns.forEach((column, columnIndex) => {
-          const uniqueValuesSet = uniqueValuesSets[columnIndex];
-          const uniqueValues = Array.from(uniqueValuesSet);
-          const containsNull = uniqueValues.includes(null) || uniqueValues.includes(void 0);
-          const symbols = uniqueValues.filter((value) => value !== null && value !== void 0).map((value) => _QvdFileWriter._convertRawToSymbol(value));
-          const currentSymbolBuffer = Buffer.concat(symbols.map((symbol) => symbol.toByteRepresentation()));
-          this._symbolBuffer = this._symbolBuffer ? Buffer.concat([this._symbolBuffer, currentSymbolBuffer]) : currentSymbolBuffer;
-          const symbolsLength = currentSymbolBuffer.length;
-          const symbolsOffset = this._symbolBuffer.length - symbolsLength;
-          this._symbolTableMetadata?.push([symbolsOffset, symbolsLength, containsNull]);
+        }
+        const columns = this._df.columns;
+        const data = this._df.data;
+        const numColumns = columns.length;
+        const numRows = data.length;
+        this._emitProgress("symbol-table", 0, numColumns);
+        const indexByValue = columns.map(() => /* @__PURE__ */ new Map());
+        const containsNull = columns.map(() => false);
+        for (let row = 0; row < numRows; row++) {
+          const values = data[row];
+          for (let column = 0; column < numColumns; column++) {
+            const value = values?.[column];
+            if (value === null || value === void 0) {
+              containsNull[column] = true;
+              continue;
+            }
+            const map = indexByValue[column];
+            if (!map.has(value)) {
+              map.set(value, map.size);
+            }
+          }
+        }
+        const columnBuffers = [];
+        let symbolsOffset = 0;
+        for (let column = 0; column < numColumns; column++) {
+          const symbols = Array.from(indexByValue[column].keys(), (value) => _QvdFileWriter._convertRawToSymbol(value));
+          const columnBuffer = Buffer.concat(symbols.map((symbol) => symbol.toByteRepresentation()));
+          columnBuffers.push(columnBuffer);
+          this._symbolTableMetadata?.push([symbolsOffset, columnBuffer.length, containsNull[column]]);
           this._symbolTable?.push(symbols);
-          this._emitProgress("symbol-table", columnIndex + 1, numColumns);
-        });
+          this._symbolIndexByValue.push(indexByValue[column]);
+          symbolsOffset += columnBuffer.length;
+          this._emitProgress("symbol-table", column + 1, numColumns);
+        }
+        this._symbolBuffer = Buffer.concat(columnBuffers);
       }
       /**
-       * Builds the index table of the QVD file.
-       *
-       * PERFORMANCE OPTIMIZATION: Uses Map-based lookups for O(1) symbol index retrieval
-       * instead of Array.findIndex() which is O(n). For columns with many unique values,
-       * this provides significant performance improvements.
+       * Builds the bit-packed index table of the QVD file.
        *
-       * Algorithm:
-       * 1. Build Map<symbol, index> for each column (O(m) where m=unique values)
-       * 2. For each data row, lookup indices in maps (O(1) per value)
-       * 3. Convert indices to bit-packed binary format
+       * A record is `recordByteSize` bytes. Each column occupies `bitWidth` consecutive bits at
+       * `bitOffset`, least significant bit first, where bit `p` of a record is
+       * `(byte[p >> 3] >> (p & 7)) & 1`. `Bias` is -2 for a column containing NULLs: stored index 0
+       * means NULL, and real symbols start at 2. This is exactly what the reader's
+       * `decodeIndexColumn` undoes, and `writeBitField` is its inverse.
        *
-       * Bit Packing:
-       * - Calculates minimum bits needed: ceil(log2(numSymbols))
-       * - Packs indices into binary string, then converts to bytes
-       * - Reduces file size significantly for columns with few unique values
+       * The previous implementation went the long way round. Per cell it converted the raw value to
+       * a QvdSymbol - a second time, the symbol table pass having already done it - to build a
+       * template-literal lookup key, then turned the resulting index into a string of bits via
+       * `toString(2).split('').map().reverse().concat().slice().reverse()`. Per row it kept an array
+       * of those strings, padded each with `padStart`, reversed and joined them, split the result
+       * with `/.{1,8}/g`, and allocated a Buffer; then `Buffer.concat` over one Buffer per row. On
+       * the 1.7M-row taxi fixture that is 34 million symbols, 34 million key strings, 34 million bit
+       * strings and 1.7 million Buffers, and it accounted for 98% of a write.
        *
-       * Example: Column with 10 unique values needs only 4 bits per value
-       * instead of 32 bits (full integer), saving 87.5% space.
+       * Now: the widths are derived from the symbol counts, the offsets by prefix sum, one Buffer is
+       * allocated for the whole table, and each value is OR-ed into place as an integer.
        *
        * @private
        */
       _buildIndexTable() {
-        this._indexTable = [];
+        assert2(this._symbolTable, "The QVD file symbol table has not been built.");
+        assert2(this._symbolTableMetadata, "The QVD file symbol table metadata has not been built.");
+        assert2(this._symbolIndexByValue, "The QVD file symbol index has not been built.");
         this._indexTableMetadata = [];
-        this._indexBuffer = Buffer.alloc(0);
-        const numRows = this._df.data.length;
+        const columns = this._df.columns;
+        const data = this._df.data;
+        const numRows = data.length;
+        const numColumns = columns.length;
+        this._recordCount = numRows;
         this._emitProgress("index-table", 0, numRows);
-        const symbolIndexMaps = this._symbolTable?.map((symbols) => {
-          const map = /* @__PURE__ */ new Map();
-          symbols.forEach((symbol, idx) => {
-            const key = `${symbol.intValue}|${symbol.doubleValue}|${symbol.stringValue}`;
-            map.set(key, idx);
-          });
-          return map;
-        });
-        let processedRows = 0;
-        const progressInterval = Math.max(1, Math.floor(numRows / 100));
-        this._df.data.forEach((row) => {
-          const indices = this._df.columns.map((column, columnIndex) => {
-            const value = row[columnIndex];
-            const symbol = _QvdFileWriter._convertRawToSymbol(value);
-            const fieldContainsNull = this._symbolTableMetadata?.[columnIndex][2];
-            if (symbol === null) {
-              return 0;
-            } else {
-              const key = `${symbol.intValue}|${symbol.doubleValue}|${symbol.stringValue}`;
-              const symbolIndex = symbolIndexMaps?.[columnIndex].get(key);
-              return fieldContainsNull ? (symbolIndex ?? 0) + 2 : symbolIndex ?? 0;
+        const layout = [];
+        let totalBits = 0;
+        for (let column = 0; column < numColumns; column++) {
+          const fieldContainsNull = this._symbolTableMetadata[column][2];
+          const symbolCount = this._symbolTable[column].length;
+          const nullShift = fieldContainsNull ? 2 : 0;
+          const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
+          const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
+          for (const index of this._symbolIndexByValue[column].values()) {
+            if (index + nullShift > maxStoredIndex) {
+              throw new QvdValidationError("The symbol table and the index table are out of sync", {
+                field: columns[column],
+                storedIndex: index + nullShift,
+                maxStoredIndex,
+                symbolCount,
+                file: this._path,
+                stage: "buildIndexTable"
+              });
             }
-          });
-          const stringIndices = indices.map((index) => {
-            const bits = _QvdFileWriter._convertInt32ToBits(index, 32);
-            let bitString = bits.join("");
-            bitString = bitString.replace(/^0+/, "") || "0";
-            return bitString;
-          });
-          this._indexTable?.push(stringIndices);
-          processedRows++;
-          if (processedRows % progressInterval === 0 || processedRows === numRows) {
-            this._emitProgress("index-table", processedRows, numRows);
           }
-        });
-        this._df.columns.forEach((column, columnIndex) => {
-          const bitOffset = this._indexTableMetadata?.slice(0, columnIndex).reduce((sum, metadata) => sum + metadata[1], 0);
-          assert(this._indexTable, "The QVD file header has not been parsed.");
-          const fieldContainsNull = this._symbolTableMetadata?.[columnIndex][2];
-          const bias = fieldContainsNull ? -2 : 0;
-          const symbolCount = this._symbolTable?.[columnIndex].length ?? 0;
-          const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + (fieldContainsNull ? 2 : 0);
-          const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
-          this._indexTableMetadata?.push([bitOffset, bitWidth, bias]);
-          this._indexTable.forEach((indices) => {
-            const bitString = indices[columnIndex];
-            assert(
-              bitString === "0" || bitString.length <= bitWidth,
-              `Stored index for column '${column}' needs ${bitString.length} bits, but BitWidth was derived as ${bitWidth} from ${symbolCount} symbols. The symbol table and the index table are out of sync.`
-            );
-            indices[columnIndex] = bitWidth === 0 ? "" : bitString.padStart(bitWidth, "0");
-          });
-        });
-        this._indexBuffer = Buffer.concat(
-          // @ts-ignore - Buffer array type compatibility
-          this._indexTable.map((indices) => {
-            indices.reverse();
-            const bits = indices.join("");
-            if (bits.length === 0) {
-              return Buffer.from(Uint8Array.from([0]));
+          layout.push({ geometry: fieldGeometry(totalBits, bitWidth), nullShift });
+          this._indexTableMetadata.push([totalBits, bitWidth, fieldContainsNull ? -2 : 0]);
+          totalBits += bitWidth;
+        }
+        const recordByteSize = Math.max(1, Math.ceil(totalBits / 8));
+        this._recordByteSize = recordByteSize;
+        this._indexBuffer = Buffer.alloc(numRows * recordByteSize);
+        const progressInterval = Math.max(1, Math.floor(numRows / 100));
+        for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
+          const values = data[row];
+          for (let column = 0; column < numColumns; column++) {
+            const value = values?.[column];
+            if (value === null || value === void 0) {
+              continue;
             }
-            const paddingWidth = (8 - bits.length % 8) % 8;
-            const paddedBits = bits.padStart(bits.length + paddingWidth, "0");
-            const bytes = paddedBits.match(/.{1,8}/g)?.map((byte) => parseInt(byte, 2));
-            bytes?.reverse();
-            assert(bytes, "Byte conversion of bit indices failed.");
-            return Buffer.from(Uint8Array.from(bytes));
-          })
-        );
-        this._recordByteSize = this._indexTable.length > 0 ? this._indexBuffer.length / this._indexTable.length : 1;
+            const index = this._symbolIndexByValue[column].get(value);
+            if (index === void 0) {
+              throw new QvdValidationError("A value is missing from the symbol table", {
+                field: columns[column],
+                row,
+                file: this._path,
+                stage: "buildIndexTable"
+              });
+            }
+            writeBitField(this._indexBuffer, recordBase, layout[column].geometry, index + layout[column].nullShift);
+          }
+          if ((row + 1) % progressInterval === 0 || row + 1 === numRows) {
+            this._emitProgress("index-table", row + 1, numRows);
+          }
+        }
+        this._symbolIndexByValue = null;
       }
       /**
        * Converts a raw value/literal to a QVD symbol.
@@ -697,16 +754,6 @@ var init_QvdFileWriter = __esm({
           return QvdSymbol.fromStringValue(raw);
         }
       }
-      /**
-       * Converts an integer to a list of bits.
-       *
-       * @param {number} value The integer value to convert.
-       * @param {number} width The width of the bit list.
-       * @return {Array<number>} The list of bits.
-       */
-      static _convertInt32ToBits(value, width) {
-        return value.toString(2).split("").map((bit) => parseInt(bit)).reverse().concat(new Array(width).fill(0)).slice(0, width).reverse();
-      }
       /**
        * Persists the data frame to a QVD file.
        */
@@ -719,18 +766,6 @@ var init_QvdFileWriter = __esm({
     };
   }
 });
-
-// src/util/bitUtils.js
-function convertBitsToInt32(bits) {
-  if (bits.length === 0) {
-    return 0;
-  }
-  return bits.reduce((value, bit, index) => value += bit * Math.pow(2, index), 0);
-}
-var init_bitUtils = __esm({
-  "src/util/bitUtils.js"() {
-  }
-});
 function getHeapLimit() {
   return v8.getHeapStatistics().heap_size_limit;
 }
@@ -740,7 +775,7 @@ function heapLimitIsMeaningful() {
 function getMemoryBudget() {
   const candidates = [];
   if (heapLimitIsMeaningful()) {
-    candidates.push({ source: "V8 heap limit", bytes: getHeapLimit() });
+    candidates.push({ source: "V8 heap limit", bytes: usableOldSpaceLimit() });
   }
   const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
   if (constrained > 0 && constrained < os.totalmem()) {
@@ -756,19 +791,56 @@ function getMemoryBudget() {
   const binding = candidates.reduce((lowest, candidate) => candidate.bytes < lowest.bytes ? candidate : lowest);
   return { bytes: binding.bytes, limitedBy: binding.source, candidates, observed };
 }
-function estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
+function usableOldSpaceLimit() {
+  const usable = getHeapLimit() - HEAP_LIMIT_OVERSTATEMENT_BYTES;
+  return usable > MINIMUM_BUDGET_BYTES ? usable : MINIMUM_BUDGET_BYTES;
+}
+function estimateExternalMemory(rows, columnCount) {
+  if (!columnCount || columnCount <= 0 || !rows || rows <= 0) {
+    return 0;
+  }
+  return rows * columnCount * Int32Array.BYTES_PER_ELEMENT;
+}
+function estimateRowMemory(rows, columnCount) {
+  if (!columnCount || columnCount <= 0) {
+    return 0;
+  }
+  return BASE_BYTES + rows * (ROW_BASE_BYTES + PER_CELL_BYTES * columnCount);
+}
+function estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
   const FULL_PARSE_OVERHEAD = 6;
   const MINIMAL_OVERHEAD = 0.01;
+  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
+  const rowMemory = materialisesRows ? estimateRowMemory(rowsToLoad, columnCount) : BASE_BYTES;
   if (maxRows === null || maxRows >= totalRows) {
-    return symbolTableSize * FULL_PARSE_OVERHEAD;
+    return symbolTableSize * FULL_PARSE_OVERHEAD + rowMemory;
   }
   const rowPercentage = maxRows / totalRows;
   const symbolPercentage = Math.sqrt(rowPercentage);
   const keptSymbolsMemory = symbolTableSize * symbolPercentage * FULL_PARSE_OVERHEAD;
   const skippedSymbolsMemory = symbolTableSize * (1 - symbolPercentage) * MINIMAL_OVERHEAD;
-  return keptSymbolsMemory + skippedSymbolsMemory;
+  return keptSymbolsMemory + skippedSymbolsMemory + rowMemory;
 }
-function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.3) {
+function recommendedRowsFor(budget, symbolTableSize, totalRows, columnCount, materialisesRows = true) {
+  if (estimateMemoryUsage(symbolTableSize, totalRows, totalRows, columnCount, materialisesRows) <= budget) {
+    return totalRows;
+  }
+  if (estimateMemoryUsage(symbolTableSize, 0, totalRows, columnCount, materialisesRows) > budget) {
+    return 0;
+  }
+  let low = 0;
+  let high = totalRows;
+  while (high - low > 1) {
+    const mid = Math.floor((low + high) / 2);
+    if (estimateMemoryUsage(symbolTableSize, mid, totalRows, columnCount, materialisesRows) <= budget) {
+      low = mid;
+    } else {
+      high = mid;
+    }
+  }
+  return low;
+}
+function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePath, safetyFactor = 0.8, columnCount = 0, materialisesRows = true) {
   if (typeof safetyFactor !== "number" || safetyFactor < 0 || safetyFactor > 1) {
     throw new QvdValidationError("safetyFactor must be a number between 0.0 and 1.0", { safetyFactor });
   }
@@ -776,25 +848,57 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
     return;
   }
   const budget = getMemoryBudget();
+  const rowsToLoad = maxRows === null || maxRows >= totalRows ? totalRows : maxRows;
+  const heapMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
+  const externalMemory = estimateExternalMemory(rowsToLoad, columnCount);
+  const bounded = budget.candidates.map((candidate) => {
+    const heapOnly = candidate.source === "V8 heap limit";
+    return {
+      ...candidate,
+      needs: heapOnly ? heapMemory : heapMemory + externalMemory,
+      allowed: candidate.bytes * safetyFactor,
+      bounds: heapOnly ? "the V8 heap" : "the whole process"
+    };
+  });
+  const exceeded = bounded.filter((candidate) => candidate.needs > candidate.allowed);
+  const binding = exceeded.reduce(
+    (worst, candidate) => candidate.needs / candidate.allowed > worst.needs / worst.allowed ? candidate : worst,
+    exceeded[0]
+  );
   const heapLimit = getHeapLimit();
-  const availableMemory = budget.bytes;
-  const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
-  const maxAllowedMemory = budget.bytes * safetyFactor;
-  if (estimatedMemory > maxAllowedMemory) {
-    const safeSymbolPercentage = maxAllowedMemory / (symbolTableSize * 6);
-    const safeRowPercentage = Math.pow(safeSymbolPercentage, 2);
-    const recommendedMaxRows = Math.max(1, Math.floor(totalRows * safeRowPercentage));
+  const availableMemory = binding ? binding.bytes : budget.bytes;
+  const estimatedMemory = binding ? binding.needs : heapMemory;
+  const maxAllowedMemory = binding ? binding.allowed : budget.bytes * safetyFactor;
+  if (binding) {
+    const recommendedMaxRows = recommendedRowsFor(
+      maxAllowedMemory,
+      symbolTableSize,
+      totalRows,
+      columnCount,
+      materialisesRows
+    );
     const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
     const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
     const availableMB = Math.round(maxAllowedMemory / 1024 / 1024);
-    const heapLimitMB = Math.round(heapLimit / 1024 / 1024);
+    const heapLimitMB = Math.round(usableOldSpaceLimit() / 1024 / 1024);
+    const reportedHeapLimitMB = Math.round(heapLimit / 1024 / 1024);
     const availableRamMB = Math.round(availableMemory / 1024 / 1024);
-    const limitingFactor = budget.limitedBy;
+    const limitingFactor = binding.source;
+    const limitingScope = binding.bounds;
     const budgetBreakdown = budget.candidates.map((candidate) => `${candidate.source} ${Math.round(candidate.bytes / 1024 / 1024)}MB`).join(", ");
     const observedBreakdown = budget.observed.map((entry) => `${entry.source} ${Math.round(entry.bytes / 1024 / 1024)}MB`).join(", ");
-    const advice = limitingFactor === "container memory limit" ? `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).` : `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
+    const nothingFits = recommendedMaxRows === 0;
+    const containerBound = binding.source === "container memory limit";
+    let advice;
+    if (nothingFits) {
+      advice = `No row count fits this budget - the symbol table alone exceeds it, so maxRows cannot help. ` + (containerBound ? `Raise the container's memory limit.` : `Raise the heap with --max-old-space-size, or raise memorySafetyFactor.`);
+    } else if (containerBound) {
+      advice = `The binding limit is the container's, so raising --max-old-space-size would let V8 grow past it and be killed by the OOM killer instead. Set it below the container limit, raise the limit, or load fewer rows with maxRows (recommended: ${recommendedMaxRows.toLocaleString()} rows or less).`;
+    } else {
+      advice = `Try loading fewer rows using the maxRows parameter (recommended: ${recommendedMaxRows.toLocaleString()} rows or less), or raise the heap with --max-old-space-size.`;
+    }
     throw new QvdValidationError(
-      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
+      `Insufficient memory to load file safely. Symbol table: ${sizeMB}MB, Estimated memory needed: ${estimatedMB}MB, Available: ${availableMB}MB (limited by ${limitingFactor}, which bounds ${limitingScope}; considered: ${budgetBreakdown}; observed but not used: ${observedBreakdown}). ` + advice,
       {
         file: filePath,
         symbolTableSize,
@@ -802,10 +906,13 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
         estimatedMemoryMB: estimatedMB,
         availableMemoryMB: availableMB,
         heapLimitMB,
+        reportedHeapLimitMB,
         availableRamMB,
         limitingFactor,
+        limitingScope,
         memoryBudget: budget.candidates,
         memoryObserved: budget.observed,
+        columnCount,
         totalRows,
         maxRows,
         recommendedMaxRows
@@ -813,12 +920,11 @@ function validateMemoryAvailability(symbolTableSize, maxRows, totalRows, filePat
     );
   }
 }
-function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows) {
-  const heapLimit = getHeapLimit();
-  const LARGE_SYMBOL_TABLE_WARNING = heapLimit * 0.125;
+function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows, columnCount = 0, materialisesRows = true) {
+  const LARGE_SYMBOL_TABLE_WARNING = usableOldSpaceLimit() * 0.125;
   if (symbolTableSize > LARGE_SYMBOL_TABLE_WARNING && (maxRows === null || maxRows >= totalRows)) {
     const sizeMB = Math.round(symbolTableSize / 1024 / 1024);
-    const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
+    const estimatedMemory = estimateMemoryUsage(symbolTableSize, maxRows, totalRows, columnCount, materialisesRows);
     const estimatedMB = Math.round(estimatedMemory / 1024 / 1024);
     const warnMB = Math.round(LARGE_SYMBOL_TABLE_WARNING / 1024 / 1024);
     console.warn(
@@ -826,13 +932,37 @@ function warnLargeSymbolTable(symbolTableSize, maxRows, totalRows) {
     );
   }
 }
+var HEAP_LIMIT_OVERSTATEMENT_BYTES, MINIMUM_BUDGET_BYTES, BASE_BYTES, ROW_BASE_BYTES, PER_CELL_BYTES;
 var init_memoryUtils = __esm({
   "src/util/memoryUtils.js"() {
     init_QvdErrors();
+    HEAP_LIMIT_OVERSTATEMENT_BYTES = 192 * 1024 * 1024;
+    MINIMUM_BUDGET_BYTES = 64 * 1024 * 1024;
+    BASE_BYTES = 16 * 1024 * 1024;
+    ROW_BASE_BYTES = 72;
+    PER_CELL_BYTES = 8;
   }
 });
 
 // src/util/validationUtils.js
+function validateHeaderStructure(headerObj, filePath, stage) {
+  const tableHeader = headerObj?.["QvdTableHeader"];
+  if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
+    throw new QvdCorruptedError("The XML header contains no usable QvdTableHeader element", {
+      rootElements: headerObj && typeof headerObj === "object" ? Object.keys(headerObj) : [],
+      file: filePath,
+      stage
+    });
+  }
+  const symbolTableLength = parseInt(tableHeader["Offset"], 10);
+  if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
+    throw new QvdCorruptedError("Invalid symbol table offset", {
+      offset: tableHeader["Offset"],
+      file: filePath,
+      stage
+    });
+  }
+}
 function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
   const heapLimit = getHeapLimit();
   const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
@@ -906,6 +1036,15 @@ function validateFieldMetadata(field, symbolBufferLength, filePath) {
     });
   }
 }
+function validateRecordCount(totalRows, filePath, stage = "parseIndexTable") {
+  if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
+    throw new QvdCorruptedError("Invalid number of records", {
+      totalRows,
+      file: filePath,
+      stage
+    });
+  }
+}
 function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, indexTableOffset, bufferLength, rowsToLoad, filePath, fileSize = null) {
   if (isNaN(recordSize) || !Number.isSafeInteger(recordSize) || recordSize < 0) {
     throw new QvdCorruptedError("Invalid record byte size", {
@@ -914,13 +1053,7 @@ function validateIndexTableMetadata(recordSize, totalRows, indexTableLength, ind
       stage: "parseIndexTable"
     });
   }
-  if (isNaN(totalRows) || !Number.isSafeInteger(totalRows) || totalRows < 0) {
-    throw new QvdCorruptedError("Invalid number of records", {
-      totalRows,
-      file: filePath,
-      stage: "parseIndexTable"
-    });
-  }
+  validateRecordCount(totalRows, filePath);
   if (recordSize === 0 && totalRows > 0) {
     throw new QvdCorruptedError("Record byte size cannot be zero when records exist", {
       recordSize,
@@ -1016,6 +1149,24 @@ function validateFieldBitMetadata(field, recordSize, filePath) {
       stage: "parseIndexTable"
     });
   }
+  const bias = parseInt(field["Bias"], 10);
+  if (isNaN(bias) || !Number.isSafeInteger(bias)) {
+    throw new QvdCorruptedError("Invalid bias", {
+      field: field["FieldName"],
+      bias: field["Bias"],
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
+  if (bitWidth > MAX_BIT_WIDTH) {
+    throw new QvdCorruptedError("Bit width exceeds maximum", {
+      field: field["FieldName"],
+      bitWidth,
+      maxBitWidth: MAX_BIT_WIDTH,
+      file: filePath,
+      stage: "parseIndexTable"
+    });
+  }
   const recordSizeInBits = recordSize * 8;
   if (bitOffset + bitWidth > recordSizeInBits) {
     throw new QvdCorruptedError("Bit field extends beyond record size", {
@@ -1032,6 +1183,7 @@ var init_validationUtils = __esm({
   "src/util/validationUtils.js"() {
     init_QvdErrors();
     init_memoryUtils();
+    init_bitUtils();
   }
 });
 
@@ -1291,6 +1443,273 @@ var init_symbolParser = __esm({
   }
 });
 
+// src/QvdColumnTable.js
+var QvdColumnTable_exports = {};
+__export(QvdColumnTable_exports, {
+  QvdColumn: () => QvdColumn,
+  QvdColumnTable: () => QvdColumnTable
+});
+var QvdColumn, QvdColumnTable;
+var init_QvdColumnTable = __esm({
+  "src/QvdColumnTable.js"() {
+    init_QvdErrors();
+    QvdColumn = class {
+      /**
+       * @param {string} name The field name.
+       * @param {Int32Array} codes One stored index per row, bias applied. Negative means NULL.
+       * @param {Array<any>} symbols The field's distinct values, indexed by code.
+       */
+      constructor(name, codes, symbols) {
+        this._name = name;
+        this._codes = codes;
+        this._symbols = symbols;
+        Object.freeze(this);
+      }
+      /** @return {string} The field name. */
+      get name() {
+        return this._name;
+      }
+      /** @return {number} Rows in the column. */
+      get length() {
+        return this._codes.length;
+      }
+      /**
+       * The stored index of each row. Negative means NULL.
+       *
+       * The table's own array, not a copy - it is the thing that makes this cheap, and copying it
+       * per call would defeat the point. Treat it as read-only.
+       *
+       * @return {Int32Array} One code per row.
+       */
+      get codes() {
+        return this._codes;
+      }
+      /**
+       * The field's distinct values, indexed by the codes.
+       *
+       * One entry per distinct value, not per row: a few thousand entries for a column of millions.
+       *
+       * @return {ReadonlyArray<any>} The dictionary.
+       */
+      get symbols() {
+        return this._symbols;
+      }
+      /**
+       * The value of one row.
+       *
+       * @param {number} row The row index.
+       * @return {any} The value, or null where the file stores NULL.
+       * @throws {QvdValidationError} If the row is not an integer within the column.
+       */
+      at(row) {
+        if (!Number.isInteger(row) || row < 0 || row >= this._codes.length) {
+          throw new QvdValidationError("Row index out of bounds", {
+            column: this._name,
+            row,
+            length: this._codes.length
+          });
+        }
+        const code = this._codes[row];
+        return code < 0 ? null : this._symbols[code];
+      }
+      /**
+       * Iterates the column's values without materialising it.
+       *
+       * @return {Iterator<any>} An iterator over the values, NULLs included as null.
+       */
+      [Symbol.iterator]() {
+        const codes = this._codes;
+        const symbols = this._symbols;
+        let row = 0;
+        return {
+          next() {
+            if (row >= codes.length) {
+              return { done: true, value: void 0 };
+            }
+            const code = codes[row++];
+            return { done: false, value: code < 0 ? null : symbols[code] };
+          }
+        };
+      }
+      /**
+       * The column as a plain array of values, one per row.
+       *
+       * Lossless, and cell-for-cell what `QvdDataFrame.data[row][column]` would hold. The caller
+       * owns the result; the table keeps no reference to it, so two calls return two arrays.
+       *
+       * @return {Array<any>} One value per row.
+       */
+      toArray() {
+        const out = new Array(this._codes.length);
+        for (let row = 0; row < this._codes.length; row++) {
+          const code = this._codes[row];
+          out[row] = code < 0 ? null : this._symbols[code];
+        }
+        return out;
+      }
+      /**
+       * The dictionary as numbers, for scanning without materialising the column.
+       *
+       * One entry per *distinct* value, not per row - 17 KB for a column of 1.7 million rows with
+       * 2,188 symbols - so this is the cheap conversion, where `toFloat64Array()` is the expensive
+       * one. Non-numeric symbols become NaN, which is safe here in a way it is not per row: the
+       * codes still distinguish NULL, and a caller that wants the blank back still has `symbols`.
+       *
+       * Scanning `codes` against this is the fastest way to read a column, because both sides are
+       * contiguous typed arrays and the dictionary fits in cache:
+       *
+       * ```js
+       * const codes = column.codes;
+       * const values = column.numericSymbols();
+       * let total = 0;
+       * for (let row = 0; row < codes.length; row++) {
+       *   const code = codes[row];
+       *   if (code >= 0) {
+       *     const value = values[code];
+       *     if (!Number.isNaN(value)) total += value;
+       *   }
+       * }
+       * ```
+       *
+       * @return {Float64Array} One number per distinct symbol, NaN where the symbol is not a number.
+       */
+      numericSymbols() {
+        const out = new Float64Array(this._symbols.length);
+        for (let index = 0; index < this._symbols.length; index++) {
+          const value = this._symbols[index];
+          out[index] = typeof value === "number" ? value : NaN;
+        }
+        return out;
+      }
+      /**
+       * The column as a `Float64Array`.
+       *
+       * Named for what it costs rather than offered as *the* representation, because it is lossy
+       * and on real data it is lossy often: it cannot distinguish a blank from a number, and the
+       * bundled taxi fixture has a column that is 43% blank. It therefore refuses by default
+       * rather than quietly writing NaN over two fifths of a column.
+       *
+       * @param {Object} [options] Conversion options.
+       * @param {'throw'|'nan'} [options.onNonNumeric='throw'] What to do with a value that is not a
+       *   number - including NULL. `'throw'` refuses and names the offending row; `'nan'` writes
+       *   NaN, which is the right choice only when the caller knows the column is numeric.
+       * @return {Float64Array} One number per row.
+       * @throws {QvdValidationError} If a value is not a number and `onNonNumeric` is `'throw'`.
+       */
+      toFloat64Array(options = {}) {
+        const { onNonNumeric = "throw" } = options;
+        if (onNonNumeric !== "throw" && onNonNumeric !== "nan") {
+          throw new QvdValidationError('onNonNumeric must be "throw" or "nan"', {
+            column: this._name,
+            provided: onNonNumeric
+          });
+        }
+        const out = new Float64Array(this._codes.length);
+        for (let row = 0; row < this._codes.length; row++) {
+          const code = this._codes[row];
+          const value = code < 0 ? null : this._symbols[code];
+          if (typeof value === "number") {
+            out[row] = value;
+            continue;
+          }
+          if (onNonNumeric === "throw") {
+            throw new QvdValidationError("Column holds a value that is not a number", {
+              column: this._name,
+              row,
+              value,
+              type: value === null ? "null" : typeof value,
+              hint: 'Pass {onNonNumeric: "nan"} to write NaN instead, or use toArray() to keep the value.'
+            });
+          }
+          out[row] = NaN;
+        }
+        return out;
+      }
+    };
+    QvdColumnTable = class {
+      /**
+       * @param {Object} decoded What the reader decoded.
+       * @param {Array<string>} decoded.columns Field names, in file order.
+       * @param {Array<Int32Array>} decoded.codesByField One code array per field.
+       * @param {Array<Array<any>>} decoded.symbolsByField One dictionary per field.
+       * @param {number} decoded.rowCount Rows decoded.
+       * @param {any} decoded.metadata The raw QvdTableHeader.
+       * @param {any} decoded.loadStats Statistics about the read.
+       */
+      constructor({ columns, codesByField, symbolsByField, rowCount, metadata, loadStats }) {
+        this._columns = columns;
+        this._codesByField = codesByField;
+        this._symbolsByField = symbolsByField;
+        this._rowCount = rowCount;
+        this._metadata = metadata;
+        this._loadStats = loadStats;
+      }
+      /**
+       * Reads a QVD file as columns.
+       *
+       * @param {string} path The path to the QVD file.
+       * @param {Object} [options] Loading options, with the same meanings they have on `fromQvd`.
+       * @param {number|null} [options.maxRows] Maximum rows to decode.
+       * @param {string} [options.allowedDir] Directory the path must resolve inside.
+       * @param {number} [options.memorySafetyFactor] Fraction of the memory budget a load may use.
+       * @param {number} [options.symbolFilteringThreshold] Symbol table size above which a limited
+       *   read switches to two-pass filtering.
+       * @return {Promise<QvdColumnTable>} The file, as columns.
+       */
+      static async fromQvd(path3, options = {}) {
+        const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
+        const reader = new QvdFileReader2(path3, {
+          allowedDir: options.allowedDir,
+          memorySafetyFactor: options.memorySafetyFactor,
+          symbolFilteringThreshold: options.symbolFilteringThreshold,
+          // This read builds no rows, so the memory guard must not charge it for them. A columnar
+          // read of the 38MB taxi fixture completes in a 15MB heap; charged the row cost it was
+          // refused below a 2GB one.
+          materialisesRows: false
+        });
+        return await reader.loadColumnar(options.maxRows !== void 0 ? options.maxRows : null);
+      }
+      /** @return {Array<string>} Field names, in file order. */
+      get columns() {
+        return this._columns;
+      }
+      /** @return {number} Rows decoded. */
+      get rowCount() {
+        return this._rowCount;
+      }
+      /** @return {Array<number>} `[rows, columns]`, as on a data frame. */
+      get shape() {
+        return [this._rowCount, this._columns.length];
+      }
+      /** @return {any} The raw `QvdTableHeader`. */
+      get metadata() {
+        return this._metadata;
+      }
+      /** @return {any} Statistics about the read. */
+      get loadStats() {
+        return this._loadStats;
+      }
+      /**
+       * One column.
+       *
+       * @param {string} name The field name.
+       * @return {QvdColumn} The column.
+       * @throws {QvdValidationError} If the field is not in this file.
+       */
+      column(name) {
+        const index = this._columns.indexOf(name);
+        if (index === -1) {
+          throw new QvdValidationError(`Column '${name}' does not exist`, {
+            column: name,
+            availableColumns: this._columns
+          });
+        }
+        return new QvdColumn(name, this._codesByField[index], this._symbolsByField[index]);
+      }
+    };
+  }
+});
+
 // src/QvdFileReader.js
 var QvdFileReader_exports = {};
 __export(QvdFileReader_exports, {
@@ -1325,13 +1744,22 @@ var init_QvdFileReader = __esm({
        *   check entirely**, which is the escape hatch for runtimes whose limits cannot be measured -
        *   Bun reports its current heap as its heap limit - and for callers who would rather manage
        *   memory themselves than trust the estimate.
+       * @param {boolean} [options.materialisesRows=true] Whether this read will build row arrays.
+       *   False for a columnar read, whose memory is TypedArray backing stores outside the V8 heap
+       *   and which must not be charged the row cost.
        * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes,
        *   above which a lazy load switches to the two-pass filtering path. The default of 50MB is
        *   the point where the extra analysis pass pays for itself; lower it to use filtering on
        *   smaller files, raise it to keep the simpler single-pass read for longer.
        */
       constructor(filePath, options = {}) {
-        const { allowedDir, memorySafetyFactor = 0.3, symbolFilteringThreshold = 50 * 1024 * 1024 } = options;
+        const {
+          allowedDir,
+          memorySafetyFactor = 0.8,
+          symbolFilteringThreshold = 50 * 1024 * 1024,
+          materialisesRows = true
+        } = options;
+        this._materialisesRows = materialisesRows;
         this._path = validatePath(filePath, allowedDir);
         this._memorySafetyFactor = memorySafetyFactor;
         this._symbolFilteringThreshold = symbolFilteringThreshold;
@@ -1341,8 +1769,10 @@ var init_QvdFileReader = __esm({
         this._indexTableOffset = null;
         this._header = null;
         this._symbolTable = null;
-        this._indexTable = null;
+        this._indexColumns = null;
+        this._rowsDecoded = 0;
         this._fileSize = null;
+        this._headerMatchesFile = false;
       }
       /**
        * Reads the binary data of the QVD file.
@@ -1369,14 +1799,13 @@ var init_QvdFileReader = __esm({
        * - Direct byte-range reading for remaining data is fastest
        *
        * @param {number|null} maxRows The maximum number of rows to load. If null, all data is loaded.
+       * @param {boolean} [headerOnly=false] Stop once the XML header has been read, leaving the
+       *   symbol and index tables on disk. This is the metadata-only path: the header is a few
+       *   kilobytes whatever the file's size, so reading a schema costs the same for a 40MB file as
+       *   for a 40GB one.
        * @private
        */
-      async _readData(maxRows = null) {
-        if (maxRows === null) {
-          this._buffer = await fs.promises.readFile(this._path);
-          this._fileSize = this._buffer.length;
-          return;
-        }
+      async _readData(maxRows = null, headerOnly = false) {
         const HEADER_DELIMITER = "\r\n\0";
         const CHUNK_SIZE = 64 * 1024;
         const stream = fs.createReadStream(this._path, {
@@ -1385,7 +1814,6 @@ var init_QvdFileReader = __esm({
         const headerChunks = [];
         let headerBytes = 0;
         let tail = Buffer.alloc(0);
-        let headerBuffer = Buffer.alloc(0);
         let headerDelimiterIndex = -1;
         try {
           for await (const chunk of stream) {
@@ -1429,7 +1857,7 @@ var init_QvdFileReader = __esm({
             }
           );
         }
-        headerBuffer = Buffer.concat(headerChunks);
+        const headerBuffer = Buffer.concat(headerChunks);
         const headerEndIndex = headerDelimiterIndex + HEADER_DELIMITER.length;
         const headerXml = headerBuffer.subarray(0, headerEndIndex).toString();
         const headerObj = await xml2.parseStringPromise(headerXml, { explicitArray: false });
@@ -1439,11 +1867,45 @@ var init_QvdFileReader = __esm({
             stage: "readData"
           });
         }
+        validateHeaderStructure(headerObj, this._path, "readData");
         const symbolTableOffset = headerEndIndex;
         const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
         const indexTableOffset = symbolTableOffset + symbolTableLength;
         const recordSize = parseInt(headerObj["QvdTableHeader"]["RecordByteSize"], 10);
         const totalRows = parseInt(headerObj["QvdTableHeader"]["NoOfRecords"], 10);
+        if (headerOnly) {
+          this._buffer = headerBuffer.subarray(0, headerEndIndex);
+          return;
+        }
+        let headerFields = headerObj["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
+        if (headerFields && !Array.isArray(headerFields)) {
+          headerFields = [headerFields];
+        }
+        const columnCount = Array.isArray(headerFields) ? headerFields.length : 0;
+        const headerNumbersUsable = [symbolTableLength, recordSize, totalRows].every(
+          (value) => Number.isSafeInteger(value) && value >= 0
+        );
+        if (headerNumbersUsable) {
+          const { size: fileSize } = await fs.promises.stat(this._path);
+          this._fileSize = fileSize;
+          this._headerMatchesFile = headerEndIndex + symbolTableLength + totalRows * recordSize <= fileSize;
+        }
+        if (headerNumbersUsable && this._headerMatchesFile) {
+          validateMemoryAvailability(
+            symbolTableLength,
+            maxRows,
+            totalRows,
+            this._path,
+            this._memorySafetyFactor,
+            columnCount,
+            this._materialisesRows
+          );
+        }
+        if (maxRows === null) {
+          this._buffer = await fs.promises.readFile(this._path);
+          this._fileSize = this._buffer.length;
+          return;
+        }
         const rowsToLoad = Math.min(maxRows, totalRows);
         validateSymbolTableSizeEarly(symbolTableLength, this._path);
         for (const [name, value] of [
@@ -1529,38 +1991,52 @@ var init_QvdFileReader = __esm({
             stage: "parseHeader"
           });
         }
+        validateHeaderStructure(this._header, this._path, "parseHeader");
+        const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
+        const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
+        if (fieldList.length === 0) {
+          throw new QvdCorruptedError("The QVD file header declares no fields", {
+            file: this._path,
+            stage: "parseHeader"
+          });
+        }
+        const malformedIndex = fieldList.findIndex(
+          (field) => field === null || typeof field !== "object" || Array.isArray(field)
+        );
+        if (malformedIndex !== -1) {
+          throw new QvdCorruptedError("The QVD file header declares a field with no properties", {
+            fieldIndex: malformedIndex,
+            fieldCount: fieldList.length,
+            file: this._path,
+            stage: "parseHeader"
+          });
+        }
         this._headerOffset = headerBeginIndex;
         this._symbolTableOffset = headerEndIndex;
         this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);
       }
       /**
-       * Estimates the memory usage for loading a QVD file with given parameters.
-       * Takes into account Phase 2.5 optimization which skips parsing unused symbols.
+       * Establishes the geometry of the index table, and validates it.
        *
-       * @param {number} symbolTableSize Size of symbol table in bytes
-       * @param {number|null} maxRows Maximum number of rows to load (null = all rows)
-       * @param {number} totalRows Total number of rows in the file
-       * @returns {number} Estimated memory usage in bytes
-       * @private
-       */
-      _estimateMemoryUsage(symbolTableSize, maxRows, totalRows) {
-        return estimateMemoryUsage(symbolTableSize, maxRows, totalRows);
-      }
-      /**
-       * Analyzes the index table to determine which symbols are actually needed.
-       * This is used for two-pass symbol filtering optimization.
+       * Both passes over the index table - the symbol-usage analysis and the decode itself - need
+       * exactly this, and they used to derive it separately with two copies of the same bit
+       * unpacking. The copies are the reason the bias comment in the analysis pass warns so loudly
+       * about keeping the sign in step with the other one: the two could drift, and #113 is what
+       * that looks like when they do. There is one copy now.
        *
-       * @param {number} maxRows The maximum number of rows to analyze.
-       * @returns {Promise<Map<string, Set<number>>>} Map of field names to Set of needed symbol indices.
+       * @param {number|null} rowLimit Maximum rows of interest, or null for all of them.
+       * @param {string} stage Stage name for any error raised here.
+       * @return {{fields: Array<any>, recordSize: number, totalRows: number, rowsToLoad: number,
+       *   indexBuffer: Buffer}} The record geometry.
        * @private
        */
-      async _analyzeIndexTableSymbolUsage(maxRows) {
+      _planIndexTable(rowLimit, stage) {
         if (!this._buffer || !this._header || !this._indexTableOffset) {
           throw new QvdCorruptedError(
             "The QVD file has not been loaded in the proper order or has not been loaded at all.",
             {
               file: this._path,
-              stage: "analyzeIndexTableSymbolUsage"
+              stage
             }
           );
         }
@@ -1570,35 +2046,58 @@ var init_QvdFileReader = __esm({
         }
         const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
         const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const rowsToAnalyze = Math.min(maxRows, totalRows);
+        const rowsToLoad = rowLimit !== null ? Math.min(rowLimit, totalRows) : totalRows;
         const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
+        validateIndexTableMetadata(
+          recordSize,
+          totalRows,
+          indexTableLength,
+          this._indexTableOffset,
+          this._buffer.length,
+          rowsToLoad,
+          this._path,
+          this._fileSize
+        );
         const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
+        for (const field of fields) {
+          validateFieldBitMetadata(field, recordSize, this._path);
+        }
+        assert2(
+          rowsToLoad === 0 || recordSize === 0 || Math.floor(indexBuffer.length / recordSize) >= rowsToLoad,
+          `The index table holds ${Math.floor(indexBuffer.length / (recordSize || 1))} whole records but ${rowsToLoad} were validated as present.`
+        );
+        return { fields, recordSize, totalRows, rowsToLoad, indexBuffer };
+      }
+      /**
+       * Analyzes the index table to determine which symbols are actually needed.
+       * This is used for two-pass symbol filtering optimization.
+       *
+       * @param {number} maxRows The maximum number of rows to analyze.
+       * @return {Promise<Map<string, Set<number>>>} Map of field names to Set of needed symbol indices.
+       * @private
+       */
+      async _analyzeIndexTableSymbolUsage(maxRows) {
+        const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "analyzeIndexTableSymbolUsage");
         const symbolUsage = /* @__PURE__ */ new Map();
+        const column = new Int32Array(rowsToLoad);
         fields.forEach((field) => {
-          symbolUsage.set(field["FieldName"], /* @__PURE__ */ new Set());
-        });
-        for (let pointer = 0, rowCount = 0; pointer < indexBuffer.length && rowCount < rowsToAnalyze; pointer += recordSize, rowCount++) {
-          const bytes = new Int32Array(indexBuffer.subarray(pointer, pointer + recordSize));
-          bytes.reverse();
-          const mask = bytes.reduce((bits, byte) => bits + ("00000000" + byte.toString(2)).slice(-8), "").split("").reverse().map((bit) => parseInt(bit));
-          fields.forEach((field) => {
-            const bitOffset = parseInt(field["BitOffset"], 10);
-            const bitWidth = parseInt(field["BitWidth"], 10);
-            const bias = parseInt(field["Bias"], 10);
-            let symbolIndex;
-            if (bitWidth === 0) {
-              symbolIndex = 0;
-            } else {
-              symbolIndex = this._convertBitsToInt32(mask.slice(bitOffset, bitOffset + bitWidth));
-            }
-            symbolIndex += bias;
-            if (symbolIndex < 0) {
-              return;
+          const needed = /* @__PURE__ */ new Set();
+          symbolUsage.set(field["FieldName"], needed);
+          decodeIndexColumn(
+            indexBuffer,
+            recordSize,
+            rowsToLoad,
+            parseInt(field["BitOffset"], 10),
+            parseInt(field["BitWidth"], 10),
+            parseInt(field["Bias"], 10),
+            column
+          );
+          for (let row = 0; row < rowsToLoad; row++) {
+            if (column[row] >= 0) {
+              needed.add(column[row]);
             }
-            const fieldName = field["FieldName"];
-            symbolUsage.get(fieldName)?.add(symbolIndex);
-          });
-        }
+          }
+        });
         return symbolUsage;
       }
       /**
@@ -1624,8 +2123,24 @@ var init_QvdFileReader = __esm({
         const symbolTableSize = symbolBuffer.length;
         const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
         validateSymbolTableSize(symbolTableSize, this._path, totalRows);
-        validateMemoryAvailability(symbolTableSize, maxRows, totalRows, this._path, this._memorySafetyFactor);
-        warnLargeSymbolTable(symbolTableSize, maxRows, totalRows);
+        if (this._headerMatchesFile) {
+          validateMemoryAvailability(
+            symbolTableSize,
+            maxRows,
+            totalRows,
+            this._path,
+            this._memorySafetyFactor,
+            Array.isArray(fields) ? fields.length : 1,
+            this._materialisesRows
+          );
+        }
+        warnLargeSymbolTable(
+          symbolTableSize,
+          maxRows,
+          totalRows,
+          Array.isArray(fields) ? fields.length : 1,
+          this._materialisesRows
+        );
         if (!Array.isArray(fields)) {
           fields = [fields];
         }
@@ -1659,93 +2174,85 @@ var init_QvdFileReader = __esm({
           return symbols;
         });
       }
-      /**
-       * Utility method to convert a bit array to an integer value.
-       *
-       * @param {Array<number>} bits The bit array
-       * @return {Number} The integer value
-       */
-      _convertBitsToInt32(bits) {
-        return convertBitsToInt32(bits);
-      }
       /**
        * Parses the bit stuffed index table of the QVD file. This method is part of the parsing process
        * and should not be called directly.
        *
+       * One `Int32Array` per field, filled by `decodeIndexColumn`, replacing an array per row filled
+       * a bit at a time. The old route, per row, built an `Int32Array` of the record's bytes,
+       * concatenated a binary string of `recordSize * 8` characters, split it into a character
+       * array, reversed that, and mapped it to one number per *bit*; then per cell it sliced the
+       * result again and summed `bit * Math.pow(2, index)`. Several arrays the length of the record
+       * in bits, built and discarded for every row.
+       *
+       * Column-major is what makes the decode tight: a field's bit offset, width and bias are the
+       * same for every row, so they are hoisted out of the loop and the inner loop does arithmetic
+       * into a typed array and nothing else. Rows are assembled later, once, in `load()`.
+       *
        * @param {number|null} maxRows The maximum number of rows to parse. If null, all rows are parsed.
        */
       async _parseIndexTable(maxRows = null) {
-        if (!this._buffer || !this._header || !this._indexTableOffset) {
-          throw new QvdCorruptedError(
-            "The QVD file has not been loaded in the proper order or has not been loaded at all.",
-            {
-              file: this._path,
-              stage: "parseIndexTable"
-            }
-          );
-        }
-        let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
+        const { fields, recordSize, rowsToLoad, indexBuffer } = this._planIndexTable(maxRows, "parseIndexTable");
+        this._rowsDecoded = rowsToLoad;
+        this._indexColumns = fields.map(
+          (field) => decodeIndexColumn(
+            indexBuffer,
+            recordSize,
+            rowsToLoad,
+            parseInt(field["BitOffset"], 10),
+            parseInt(field["BitWidth"], 10),
+            parseInt(field["Bias"], 10),
+            new Int32Array(rowsToLoad)
+          )
+        );
+      }
+      /**
+       * Reads the file's schema and header metadata, without touching the symbol or index tables.
+       *
+       * Constant cost in the size of the file. `fromQvd(path, {maxRows: 0})` is not the same thing
+       * and never was: the lazy path reads `headerEnd + symbolTableLength + rowsToLoad * recordSize`
+       * bytes, so it still pulls the entire symbol table - 0.4MB on the taxi fixture, but hundreds
+       * of megabytes on a high-cardinality file, and it has to be parsed as well as read.
+       *
+       * The memory check is deliberately not run for this. It sizes the rows a call will
+       * materialise, and this materialises none; applying it would let a file too large to load
+       * refuse to say what is in it.
+       *
+       * @return {Promise<import('./QvdDataFrame.js').QvdFileMetadata>} The file's schema and header.
+       */
+      async loadMetadata() {
+        await this._readData(null, true);
+        await this._parseHeader();
+        assert2(this._header, "The QVD file header has not been parsed.");
+        const header = this._header["QvdTableHeader"];
+        let fields = header["Fields"]?.["QvdFieldHeader"] ?? [];
         if (!Array.isArray(fields)) {
           fields = [fields];
         }
-        const recordSize = parseInt(this._header["QvdTableHeader"]["RecordByteSize"], 10);
-        const totalRows = parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10);
-        const rowsToLoad = maxRows !== null ? Math.min(maxRows, totalRows) : totalRows;
-        const indexTableLength = parseInt(this._header["QvdTableHeader"]["Length"], 10);
-        validateIndexTableMetadata(
-          recordSize,
-          totalRows,
-          indexTableLength,
-          this._indexTableOffset,
-          this._buffer.length,
-          rowsToLoad,
-          this._path,
-          this._fileSize
-        );
-        const indexBuffer = this._buffer.subarray(this._indexTableOffset, this._indexTableOffset + indexTableLength + 1);
-        for (const field of fields) {
-          validateFieldBitMetadata(field, recordSize, this._path);
-        }
-        this._indexTable = [];
-        for (let pointer = 0, rowCount = 0; pointer < indexBuffer.length && rowCount < rowsToLoad; pointer += recordSize, rowCount++) {
-          if (pointer + recordSize > indexBuffer.length) {
-            throw new QvdCorruptedError("Buffer overflow reading index table record", {
-              pointer,
-              recordSize,
-              bufferSize: indexBuffer.length,
-              file: this._path,
-              stage: "parseIndexTable"
-            });
-          }
-          const bytes = new Int32Array(indexBuffer.subarray(pointer, pointer + recordSize));
-          bytes.reverse();
-          const mask = bytes.reduce((bits, byte) => bits + ("00000000" + byte.toString(2)).slice(-8), "").split("").reverse().map((bit) => parseInt(bit));
-          const symbolIndices = [];
-          fields.forEach((field) => {
-            const bitOffset = parseInt(field["BitOffset"], 10);
-            const bitWidth = parseInt(field["BitWidth"], 10);
-            const bias = parseInt(field["Bias"], 10);
-            let symbolIndex;
-            if (bitWidth === 0) {
-              symbolIndex = 0;
-            } else {
-              symbolIndex = this._convertBitsToInt32(mask.slice(bitOffset, bitOffset + bitWidth));
-            }
-            symbolIndex += bias;
-            symbolIndices.push(symbolIndex);
-          });
-          this._indexTable.push(symbolIndices);
-        }
-        if (this._indexTable.length !== rowsToLoad) {
-          throw new QvdCorruptedError("Index table contains fewer records than expected", {
-            expectedRows: rowsToLoad,
-            actualRows: this._indexTable.length,
-            totalRows,
-            recordSize,
+        if (fields.length === 0) {
+          throw new QvdCorruptedError("The QVD file header declares no fields", {
             file: this._path,
-            stage: "parseIndexTable"
+            stage: "readMetadata"
           });
         }
+        const columns = fields.map((field) => field["FieldName"]);
+        const rowCount = parseInt(header["NoOfRecords"], 10);
+        validateRecordCount(rowCount, this._path, "readMetadata");
+        const shape = new QvdDataFrame([], columns, header, {
+          symbolTableBytes: parseInt(header["Offset"], 10),
+          totalRows: rowCount,
+          rowsLoaded: 0,
+          symbolFiltering: false,
+          symbolsKept: null
+        });
+        return {
+          columns,
+          rowCount,
+          columnCount: columns.length,
+          fields: columns.map((name) => shape.getFieldMetadata(name)),
+          fileMetadata: shape.fileMetadata,
+          metadata: header
+        };
       }
       /**
        * Loads the QVD file into memory and parses it.
@@ -1756,6 +2263,37 @@ var init_QvdFileReader = __esm({
        * @return {Promise<QvdDataFrame>} The loaded QVD file.
        */
       async load(maxRows = null) {
+        const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
+        assert2(this._indexColumns, "The QVD file index table has not been parsed.");
+        const indexColumns = this._indexColumns;
+        const fieldCount = indexColumns.length;
+        const data = new Array(this._rowsDecoded);
+        for (let row = 0; row < this._rowsDecoded; row++) {
+          const values = new Array(fieldCount);
+          for (let field = 0; field < fieldCount; field++) {
+            const symbolIndex = indexColumns[field][row];
+            values[field] = symbolIndex < 0 ? null : resolvedByField[field][symbolIndex];
+          }
+          data[row] = values;
+        }
+        loadStats.rowsLoaded = data.length;
+        return new QvdDataFrame(data, columns, metadata, loadStats);
+      }
+      /**
+       * Reads and decodes the file, stopping short of building rows.
+       *
+       * Everything `load()` and `loadColumnar()` have in common, which is everything except the
+       * shape of the answer. Two read paths for one binary format is the drift risk #113 is the
+       * standing example of - a stored index resolved one way here and another way there returns
+       * plausible wrong values and throws nothing - so there is one path, and the two entry points
+       * differ only in what they do with what it returns.
+       *
+       * @param {number|null} maxRows Maximum rows to decode, or null for all of them.
+       * @return {Promise<{columns: Array<string>, metadata: any, loadStats: any,
+       *   resolvedByField: Array<Array<any>>}>} The decoded file.
+       * @private
+       */
+      async _decode(maxRows = null) {
         if (maxRows !== null && (typeof maxRows !== "number" || !Number.isInteger(maxRows) || maxRows < 0)) {
           throw new QvdValidationError("maxRows must be a non-negative integer, or null to load all rows", {
             provided: maxRows,
@@ -1776,46 +2314,57 @@ var init_QvdFileReader = __esm({
         }
         await this._parseSymbolTable(symbolsToKeep, maxRows);
         await this._parseIndexTable(maxRows);
-        assert(this._header, "The QVD file header has not been parsed.");
-        assert(this._symbolTable, "The QVD file symbol table has not been parsed.");
-        assert(this._indexTable, "The QVD file index table has not been parsed.");
-        const getRow = (index) => {
-          if (!this._indexTable || index >= this._indexTable.length) {
-            throw new QvdValidationError("Row index out of bounds", {
-              index,
-              max: this._indexTable ? this._indexTable.length - 1 : -1,
-              file: this._path
-            });
+        assert2(this._header, "The QVD file header has not been parsed.");
+        assert2(this._symbolTable, "The QVD file symbol table has not been parsed.");
+        assert2(this._indexColumns, "The QVD file index table has not been parsed.");
+        const resolvedByField = this._symbolTable.map((symbols) => {
+          const resolved = new Array(symbols.length);
+          for (let index = 0; index < symbols.length; index++) {
+            const value = symbols[index]?.toPrimaryValue();
+            resolved[index] = typeof value === "string" && value.trim() !== "" && !isNaN(Number(value)) ? Number(value) : value;
           }
-          return this._indexTable?.[index].map((symbolIndex, fieldIndex) => {
-            if (symbolIndex < 0) {
-              return null;
-            }
-            const symbol = this._symbolTable?.[fieldIndex]?.[symbolIndex];
-            const value = symbol?.toPrimaryValue();
-            if (typeof value === "string") {
-              if (value.trim() !== "" && !isNaN(Number(value))) {
-                return Number(value);
-              }
-            }
-            return value;
-          });
-        };
+          return resolved;
+        });
         let fields = this._header["QvdTableHeader"]["Fields"]["QvdFieldHeader"];
         if (!Array.isArray(fields)) {
           fields = [fields];
         }
         const columns = fields.map((field) => field["FieldName"]);
-        const data = this._indexTable.map((_, index) => getRow(index));
         const metadata = this._header["QvdTableHeader"];
         const loadStats = {
           symbolTableBytes: parseInt(this._header["QvdTableHeader"]["Offset"], 10),
           totalRows: parseInt(this._header["QvdTableHeader"]["NoOfRecords"], 10),
-          rowsLoaded: data.length,
+          rowsLoaded: this._rowsDecoded,
           symbolFiltering: symbolsToKeep !== null,
           symbolsKept
         };
-        return new QvdDataFrame(data, columns, metadata, loadStats);
+        return { columns, metadata, loadStats, resolvedByField };
+      }
+      /**
+       * Reads the file as columns, without ever materialising rows.
+       *
+       * Shares every step with `load()` up to the point where rows would be built - see `_decode`.
+       * What it keeps instead is what the decoder already produced: one `Int32Array` of stored
+       * indices per field, and one resolved value per distinct symbol. On the 1.7M x 20 taxi
+       * fixture that is 38.6 MiB against the 352.8 MiB `data` retains, because a column costs four
+       * bytes per row rather than a boxed value per cell, and the symbols are a few thousand
+       * entries shared across every row that uses them.
+       *
+       * @param {number|null} maxRows The maximum number of rows to decode.
+       * @return {Promise<import('./QvdColumnTable.js').QvdColumnTable>} The decoded columns.
+       */
+      async loadColumnar(maxRows = null) {
+        const { columns, metadata, loadStats, resolvedByField } = await this._decode(maxRows);
+        const { QvdColumnTable: QvdColumnTable2 } = await Promise.resolve().then(() => (init_QvdColumnTable(), QvdColumnTable_exports));
+        assert2(this._indexColumns, "The QVD file index table has not been parsed.");
+        return new QvdColumnTable2({
+          columns,
+          codesByField: this._indexColumns,
+          symbolsByField: resolvedByField,
+          rowCount: this._rowsDecoded,
+          metadata,
+          loadStats
+        });
       }
     };
   }
@@ -2255,10 +2804,11 @@ var init_QvdDataFrame = __esm({
        *   outside it is rejected. Defaults to the current working directory. To permit an entire
        *   volume, pass its root explicitly ('/' on POSIX, 'C:\\' on Windows); a null or empty value falls
        *   back to the working directory rather than removing the restriction.
-       * @param {number} [options.memorySafetyFactor=0.3] Fraction (0.0-1.0) of the memory budget a load
-       *   may use. The budget is the smallest of the V8 heap limit, any container memory limit, and the
-       *   memory the OS reports as available. Default is 0.3; raise it when running with a larger heap via
-       *   --max-old-space-size. **Zero disables the memory check entirely.**
+       * @param {number} [options.memorySafetyFactor=0.8] Fraction (0.0-1.0) of the memory budget a load
+       *   may use. The budget is the smaller of the V8 heap limit and any container memory limit. Default
+       *   is 0.8; the estimate it scales accounts for the rows and columns being materialised, so this is
+       *   headroom for garbage collection rather than compensation for an inaccurate figure.
+       *   **Zero disables the memory check entirely.**
        * @param {number} [options.symbolFilteringThreshold=52428800] Symbol table size, in bytes, above which
        *   a lazy load switches to the two-pass filtering path. Defaults to 50MB.
        * @throws {QvdValidationError} If options.maxRows is neither null/undefined nor a non-negative integer.
@@ -2273,6 +2823,34 @@ var init_QvdDataFrame = __esm({
         };
         return await new QvdFileReader2(path3, readerOptions).load(options.maxRows !== void 0 ? options.maxRows : null);
       }
+      /**
+       * Reads a QVD file's schema and header metadata, without reading its data.
+       *
+       * Costs the same whatever the file's size, because it stops at the XML header - a few
+       * kilobytes - and never touches the symbol or index tables.
+       *
+       * This is what `{maxRows: 0}` looks like but is not. That still reads and parses the whole
+       * symbol table, which is 0.4MB on a 38MB taxi fixture but hundreds of megabytes on a
+       * high-cardinality file. Use this when you want to know what is in a file rather than to
+       * read any of it.
+       *
+       * No data frame comes back, deliberately: one with `data: []` would be indistinguishable
+       * from an empty file at the call site.
+       *
+       * ```js
+       * const {columns, rowCount, fields} = await QvdDataFrame.readMetadata('sales.qvd');
+       * ```
+       *
+       * @param {string} path The path to the QVD file.
+       * @param {Object} [options] Optional reading options.
+       * @param {string} [options.allowedDir] Optional allowed directory path, applied exactly as it
+       *   is for `fromQvd`.
+       * @return {Promise<QvdFileMetadata>} The file's schema and header metadata.
+       */
+      static async readMetadata(path3, options = {}) {
+        const { QvdFileReader: QvdFileReader2 } = await Promise.resolve().then(() => (init_QvdFileReader(), QvdFileReader_exports));
+        return await new QvdFileReader2(path3, { allowedDir: options.allowedDir }).loadMetadata();
+      }
       /**
        * Constructs a data frame from a dictionary.
        *
@@ -2299,10 +2877,11 @@ var init_QvdDataFrame = __esm({
 // src/index.js
 init_QvdSymbol();
 init_QvdDataFrame();
+init_QvdColumnTable();
 init_QvdFileReader();
 init_QvdFileWriter();
 init_QvdErrors();
 
-export { QvdCorruptedError, QvdDataFrame, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError };
+export { QvdColumn, QvdColumnTable, QvdCorruptedError, QvdDataFrame, QvdError, QvdFileReader, QvdFileWriter, QvdIOError, QvdParseError, QvdSecurityError, QvdSymbol, QvdValidationError };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map